16:07:36.514 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
16:07:36.526 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskmIbLz2/test-tmp
16:07:36.529 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
16:07:36.533 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskmIbLz2/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
16:07:36.791 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
16:07:36.954 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:07:36.955 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:07:36.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:36.960073 374 mini_oidc.cc:152] Starting JWKS server
16:07:36.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:36.963882 374 webserver.cc:460] Webserver started at http://127.0.0.1:36513/ using document root <none> and password file <none>
16:07:36.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:36.964454 374 mini_oidc.cc:189] Starting OIDC Discovery server
16:07:36.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:36.966691 374 webserver.cc:460] Webserver started at http://127.0.0.1:45901/ using document root <none> and password file <none>
16:07:37.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T16:07:36Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
16:07:37.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T16:07:37Z Disabled control of system clock
16:07:37.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:37.019716 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:37.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:37.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-0/wal
16:07:37.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-0/data
16:07:37.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:07:37.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:07:37.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:07:37.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:37.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:37.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:37.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-0/logs
16:07:37.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-0/data/info.pb
16:07:37.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:37.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:07:37.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:07:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:07:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:07:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:07:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:07:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:07:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:07:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:43221
16:07:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
16:07:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:07:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35119
16:07:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:37.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:37.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:07:37.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:07:37.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:43221,127.0.93.189:45349,127.0.93.188:44501
16:07:37.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:07:37.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36513/jwks.json with env {}
16:07:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:37.351227 386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:07:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:37.351783 386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:07:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:37.352459 386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:07:37.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:37.383908 386 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:07:37.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:37.384204 386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:07:37.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:37.384395 386 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:07:37.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:37.384573 386 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:07:37.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:37.384868 386 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:07:37.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:37.385084 386 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36513/jwks.json
16:07:37.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:37.422353 386 master_runner.cc:386] Master server non-default flags:
16:07:37.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:37.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35119
16:07:37.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:37.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:07:37.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-0/data
16:07:37.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-0/wal
16:07:37.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:07:37.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:43221,127.0.93.189:45349,127.0.93.188:44501
16:07:37.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:37.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:37.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:07:37.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:43221
16:07:37.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:07:37.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:07:37.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:07:37.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36513/jwks.json
16:07:37.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:37.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:37.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-0/data/info.pb
16:07:37.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
16:07:37.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:07:37.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:07:37.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:37.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:07:37.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:07:37.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:37.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-0/logs
16:07:37.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:37.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:07:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:07:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:07:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:07:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:07:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:07:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:07:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:07:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:07:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:37.423754 386 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:07:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:37.426033 386 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:07:37.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:37.449234 395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:38.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:37.449235 393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:38.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:37.449267 392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:38.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:37.451447 386 server_base.cc:1034] running on GCE node
16:07:38.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.679836 386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:07:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.683115 386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:07:38.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.684711 386 hybrid_clock.cc:648] HybridClock initialized: now 1790438858684669 us; error 58 us; skew 500 ppm
16:07:38.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.685647 386 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:07:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.693297 386 webserver.cc:460] Webserver started at http://127.0.93.190:44191/ using document root <none> and password file <none>
16:07:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.694365 386 fs_manager.cc:359] Metadata directory not provided
16:07:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.694566 386 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:07:38.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.694967 386 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:07:38.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.700209 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-0/data/instance:
16:07:38.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5"
16:07:38.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:38 on dist-test-slave-50w2"
16:07:38.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.701558 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-0/wal/instance:
16:07:38.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5"
16:07:38.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:38 on dist-test-slave-50w2"
16:07:38.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.709766 386 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
16:07:38.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.716097 402 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:07:38.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.717615 386 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
16:07:38.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.718019 386 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-0/data,/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-0/wal
16:07:38.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5"
16:07:38.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:38 on dist-test-slave-50w2"
16:07:38.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.718482 386 fs_report.cc:383] FS layout report
16:07:38.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:07:38.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-0/wal
16:07:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-0/wal
16:07:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-0/data/data
16:07:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:07:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:07:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:07:38.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:07:38.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:07:38.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:07:38.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:07:38.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:07:38.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:07:38.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:07:38.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:07:38.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.760183 386 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:07:38.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.761883 386 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:07:38.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.762452 386 kserver.cc:163] Server-wide thread pool size limit: 3276
16:07:38.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.843456 386 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.190:43221
16:07:38.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.843564 453 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.190:43221 every 8 connection(s)
16:07:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.846457 386 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-0/data/info.pb
16:07:38.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.852684 454 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:07:38.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.853106 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu as pid 386
16:07:38.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.853571 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-0/wal/instance
16:07:38.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.856871 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:38.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:38.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-1/wal
16:07:38.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-1/data
16:07:38.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:07:38.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:07:38.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:07:38.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:38.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:38.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:38.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-1/logs
16:07:38.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-1/data/info.pb
16:07:38.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:38.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:07:38.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:07:38.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:07:38.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:07:38.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:38.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:38.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:38.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:07:38.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:07:38.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:07:38.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:07:38.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:45349
16:07:38.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
16:07:38.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:07:38.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35119
16:07:38.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:07:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:07:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:43221,127.0.93.189:45349,127.0.93.188:44501
16:07:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:07:38.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36513/jwks.json with env {}
16:07:38.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.867287 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } has no permanent_uuid. Determining permanent_uuid...
16:07:38.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.902295 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } has no permanent_uuid. Determining permanent_uuid...
16:07:38.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:38.904735 403 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.189:45349: connect: Connection refused (error 111)
16:07:38.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:38.910169 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:45349: Network error: Client connection negotiation failed: client connection to 127.0.93.189:45349: connect: Connection refused (error 111)
16:07:38.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:38.959710 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } attempt: 1
16:07:38.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:38.963755 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:45349: Network error: Client connection negotiation failed: client connection to 127.0.93.189:45349: connect: Connection refused (error 111)
16:07:39.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:39.032668 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } attempt: 2
16:07:39.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:39.036935 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:45349: Network error: Client connection negotiation failed: client connection to 127.0.93.189:45349: connect: Connection refused (error 111)
16:07:39.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:39.128509 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } attempt: 3
16:07:39.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:39.134123 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:45349: Network error: Client connection negotiation failed: client connection to 127.0.93.189:45349: connect: Connection refused (error 111)
16:07:39.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:39.191057 456 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:07:39.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:39.191638 456 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:07:39.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:39.192207 456 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:07:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:39.224525 456 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:07:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:39.224829 456 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:07:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:39.225015 456 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:07:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:39.225193 456 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:07:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:39.225364 456 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:07:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:39.225621 456 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36513/jwks.json
16:07:39.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:39.261677 456 master_runner.cc:386] Master server non-default flags:
16:07:39.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:39.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35119
16:07:39.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:39.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:07:39.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-1/data
16:07:39.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-1/wal
16:07:39.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:07:39.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:43221,127.0.93.189:45349,127.0.93.188:44501
16:07:39.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:39.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:39.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:07:39.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:45349
16:07:39.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:07:39.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:07:39.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:07:39.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36513/jwks.json
16:07:39.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:39.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:39.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-1/data/info.pb
16:07:39.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
16:07:39.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:07:39.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:07:39.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:39.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:07:39.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:07:39.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:39.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-1/logs
16:07:39.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:39.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:07:39.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:07:39.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:07:39.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:07:39.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:07:39.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:07:39.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:07:39.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:07:39.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:07:39.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:39.262948 456 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:07:39.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:39.265080 456 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:07:39.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:39.277596 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } attempt: 4
16:07:39.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:39.282565 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:45349: Network error: Client connection negotiation failed: client connection to 127.0.93.189:45349: connect: Connection refused (error 111)
16:07:39.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:39.286450 465 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:39.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:39.582168 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } attempt: 5
16:07:39.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:39.588120 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:45349: Network error: Client connection negotiation failed: client connection to 127.0.93.189:45349: connect: Connection refused (error 111)
16:07:40.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.135802 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } attempt: 6
16:07:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:40.144272 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:45349: Network error: Client connection negotiation failed: client connection to 127.0.93.189:45349: connect: Connection refused (error 111)
16:07:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:39.287735 468 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:39.289289 456 server_base.cc:1034] running on GCE node
16:07:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:39.286538 466 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:40.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.527964 456 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:07:40.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.531075 456 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:07:40.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.532596 456 hybrid_clock.cc:648] HybridClock initialized: now 1790438860532568 us; error 86 us; skew 500 ppm
16:07:40.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.533977 456 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:07:40.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.541368 456 webserver.cc:460] Webserver started at http://127.0.93.189:42845/ using document root <none> and password file <none>
16:07:40.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.542569 456 fs_manager.cc:359] Metadata directory not provided
16:07:40.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.542774 456 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:07:40.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.543202 456 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:07:40.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.548771 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-1/data/instance:
16:07:40.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d8d0a877704d48fda9faeea018afe895"
16:07:40.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:40 on dist-test-slave-50w2"
16:07:40.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.550200 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-1/wal/instance:
16:07:40.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d8d0a877704d48fda9faeea018afe895"
16:07:40.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:40 on dist-test-slave-50w2"
16:07:40.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.559080 456 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
16:07:40.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.566280 477 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:07:40.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.567663 456 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
16:07:40.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.568002 456 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-1/data,/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-1/wal
16:07:40.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d8d0a877704d48fda9faeea018afe895"
16:07:40.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:40 on dist-test-slave-50w2"
16:07:40.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.568336 456 fs_report.cc:383] FS layout report
16:07:40.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:07:40.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-1/wal
16:07:40.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-1/wal
16:07:40.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-1/data/data
16:07:40.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:07:40.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:07:40.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:07:40.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:07:40.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:07:40.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:07:40.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:07:40.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:07:40.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:07:40.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:07:40.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:07:40.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.612895 456 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:07:40.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.614764 456 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:07:40.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.615257 456 kserver.cc:163] Server-wide thread pool size limit: 3276
16:07:40.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.703431 456 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.189:45349
16:07:40.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.703513 528 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.189:45349 every 8 connection(s)
16:07:40.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.706401 456 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-1/data/info.pb
16:07:40.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.710531 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu as pid 456
16:07:40.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.711028 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-1/wal/instance
16:07:40.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.713677 529 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:07:40.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.714522 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-2/wal
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-2/data
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-2/logs
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-2/data/info.pb
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:07:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:07:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:07:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:44501
16:07:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
16:07:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:07:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35119
16:07:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:07:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:07:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:43221,127.0.93.189:45349,127.0.93.188:44501
16:07:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:07:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36513/jwks.json with env {}
16:07:40.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.728128 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } has no permanent_uuid. Determining permanent_uuid...
16:07:40.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.753893 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } has no permanent_uuid. Determining permanent_uuid...
16:07:40.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.772518 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } has no permanent_uuid. Determining permanent_uuid...
16:07:40.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:40.775254 480 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44501: connect: Connection refused (error 111)
16:07:40.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:40.780356 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44501: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44501: connect: Connection refused (error 111)
16:07:40.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.829931 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } attempt: 1
16:07:40.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:40.834161 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44501: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44501: connect: Connection refused (error 111)
16:07:40.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.902662 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } attempt: 2
16:07:40.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:40.907603 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44501: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44501: connect: Connection refused (error 111)
16:07:40.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:40.999091 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } attempt: 3
16:07:41.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.003182 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44501: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44501: connect: Connection refused (error 111)
16:07:41.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.045265 531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:07:41.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.046330 531 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:07:41.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.047049 531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:07:41.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.078622 531 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:07:41.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.079012 531 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:07:41.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.079340 531 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:07:41.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.079731 531 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:07:41.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.080075 531 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:07:41.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.080304 531 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36513/jwks.json
16:07:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:41.116150 531 master_runner.cc:386] Master server non-default flags:
16:07:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35119
16:07:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:07:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-2/data
16:07:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-2/wal
16:07:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:07:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:43221,127.0.93.189:45349,127.0.93.188:44501
16:07:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:07:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:44501
16:07:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:07:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:07:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:07:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36513/jwks.json
16:07:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-2/data/info.pb
16:07:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
16:07:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:07:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:07:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:07:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:07:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-2/logs
16:07:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:07:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:07:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:07:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:07:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:07:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:07:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:07:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:07:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:07:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:41.117854 531 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:07:41.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:41.119997 531 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:07:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.143523 542 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:41.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:41.146625 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } attempt: 4
16:07:41.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.152588 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44501: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44501: connect: Connection refused (error 111)
16:07:41.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:41.204974 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } attempt: 7
16:07:41.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:41.243810 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } has no permanent_uuid. Determining permanent_uuid...
16:07:41.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.265826 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44501: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44501: connect: Connection refused (error 111)
16:07:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:41.324546 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } attempt: 1
16:07:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.331907 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44501: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44501: connect: Connection refused (error 111)
16:07:41.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:41.413733 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } attempt: 2
16:07:41.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.420338 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44501: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44501: connect: Connection refused (error 111)
16:07:41.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:41.452155 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } attempt: 5
16:07:41.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.458346 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44501: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44501: connect: Connection refused (error 111)
16:07:41.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:41.506234 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } attempt: 3
16:07:41.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.516577 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44501: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44501: connect: Connection refused (error 111)
16:07:41.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:41.657289 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } attempt: 4
16:07:41.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.664251 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44501: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44501: connect: Connection refused (error 111)
16:07:41.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:41.947834 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } attempt: 5
16:07:41.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.954367 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44501: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44501: connect: Connection refused (error 111)
16:07:42.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.005982 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } attempt: 6
16:07:42.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:42.018177 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44501: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44501: connect: Connection refused (error 111)
16:07:42.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.143790 541 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:42.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:41.144749 544 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:42.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.452968 531 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:07:42.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:42.453519 543 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1307 milliseconds
16:07:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.455674 531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:07:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.458967 531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:07:42.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.460441 531 hybrid_clock.cc:648] HybridClock initialized: now 1790438862460375 us; error 81 us; skew 500 ppm
16:07:42.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.461691 531 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:07:42.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.472654 531 webserver.cc:460] Webserver started at http://127.0.93.188:41759/ using document root <none> and password file <none>
16:07:42.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.473793 531 fs_manager.cc:359] Metadata directory not provided
16:07:42.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.474037 531 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:07:42.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.474596 531 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:07:42.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.479658 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-2/data/instance:
16:07:42.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "896fb523ff8c4a1f8c24b043bbdd1507"
16:07:42.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:42 on dist-test-slave-50w2"
16:07:42.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.481071 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-2/wal/instance:
16:07:42.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "896fb523ff8c4a1f8c24b043bbdd1507"
16:07:42.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:42 on dist-test-slave-50w2"
16:07:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.489127 531 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
16:07:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.495498 555 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:07:42.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.496677 531 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
16:07:42.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.497061 531 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-2/data,/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-2/wal
16:07:42.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "896fb523ff8c4a1f8c24b043bbdd1507"
16:07:42.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:42 on dist-test-slave-50w2"
16:07:42.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.497556 531 fs_report.cc:383] FS layout report
16:07:42.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:07:42.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-2/wal
16:07:42.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-2/wal
16:07:42.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-2/data/data
16:07:42.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:07:42.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:07:42.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:07:42.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:07:42.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:07:42.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:07:42.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:07:42.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:07:42.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:07:42.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:07:42.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:07:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.507030 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } attempt: 6
16:07:42.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:42.515115 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44501: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44501: connect: Connection refused (error 111)
16:07:42.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.540531 531 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:07:42.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.542109 531 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:07:42.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.542660 531 kserver.cc:163] Server-wide thread pool size limit: 3276
16:07:42.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.627254 531 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.188:44501
16:07:42.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.627362 608 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.188:44501 every 8 connection(s)
16:07:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.630287 531 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-2/data/info.pb
16:07:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.633529 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu as pid 531
16:07:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.634004 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/master-2/wal/instance
16:07:42.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.637815 609 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:07:42.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.656615 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } has no permanent_uuid. Determining permanent_uuid...
16:07:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.686245 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } has no permanent_uuid. Determining permanent_uuid...
16:07:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.697974 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } has no permanent_uuid. Determining permanent_uuid...
16:07:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.721966 609 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507: Bootstrap starting.
16:07:42.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.731315 609 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507: Neither blocks nor log segments found. Creating new log.
16:07:42.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.733610 609 log.cc:824] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507: Log is configured to *not* fsync() on all Append() calls
16:07:42.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.740136 609 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507: No bootstrap required, opened a new log
16:07:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.762264 609 raft_consensus.cc:348] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } }
16:07:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.762915 609 raft_consensus.cc:374] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:07:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.763110 609 raft_consensus.cc:729] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 896fb523ff8c4a1f8c24b043bbdd1507, State: Initialized, Role: FOLLOWER
16:07:42.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.763743 609 consensus_queue.cc:260] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } }
16:07:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.766394 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } } }
16:07:42.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.767791 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [sys.catalog]: This master's current role is: FOLLOWER
16:07:42.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.768167 609 sys_catalog.cc:564] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [sys.catalog]: configured and running, proceeding with master startup.
16:07:42.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:42.797392 629 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:07:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:42.797809 629 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:07:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:42.798848 563 tablet.cc:2369] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:07:42.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.793269 531 master_runner.cc:186] Error getting master registration for 127.0.93.190:43221: OK, instance_id { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" instance_seqno: 1790438858821809 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:07:42.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:42.820456 531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:07:43.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.058666 618 raft_consensus.cc:482] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:07:43.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.059105 618 raft_consensus.cc:504] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } }
16:07:43.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.061388 618 leader_election.cc:288] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0b66f7ad88c04ed98e89a3f1d8314bd5 (127.0.93.190:43221), d8d0a877704d48fda9faeea018afe895 (127.0.93.189:45349)
16:07:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.078732 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } attempt: 7
16:07:43.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.102584 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895: Bootstrap starting.
16:07:43.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.110263 529 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895: Neither blocks nor log segments found. Creating new log.
16:07:43.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.112653 529 log.cc:824] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895: Log is configured to *not* fsync() on all Append() calls
16:07:43.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.119601 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895: No bootstrap required, opened a new log
16:07:43.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.152428 529 raft_consensus.cc:348] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } }
16:07:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.153337 529 raft_consensus.cc:374] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:07:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.153644 529 raft_consensus.cc:729] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d8d0a877704d48fda9faeea018afe895, State: Initialized, Role: FOLLOWER
16:07:43.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.154582 529 consensus_queue.cc:260] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } }
16:07:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.157572 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } } }
16:07:43.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.158766 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895 [sys.catalog]: This master's current role is: FOLLOWER
16:07:43.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.159442 529 sys_catalog.cc:564] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895 [sys.catalog]: configured and running, proceeding with master startup.
16:07:43.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.160105 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d8d0a877704d48fda9faeea018afe895" is_pre_election: true
16:07:43.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.161123 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 896fb523ff8c4a1f8c24b043bbdd1507 in term 0.
16:07:43.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.162568 558 leader_election.cc:302] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 896fb523ff8c4a1f8c24b043bbdd1507, d8d0a877704d48fda9faeea018afe895; no voters:
16:07:43.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.164187 618 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [term 0 FOLLOWER]: Leader pre-election won for term 1
16:07:43.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.164588 618 raft_consensus.cc:482] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:07:43.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.164901 618 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [term 0 FOLLOWER]: Advancing to term 1
16:07:43.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.172426 618 raft_consensus.cc:504] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } }
16:07:43.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.183107 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d8d0a877704d48fda9faeea018afe895"
16:07:43.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.183703 504 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895 [term 0 FOLLOWER]: Advancing to term 1
16:07:43.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.189160 618 leader_election.cc:288] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [CANDIDATE]: Term 1 election: Requested vote from peers 0b66f7ad88c04ed98e89a3f1d8314bd5 (127.0.93.190:43221), d8d0a877704d48fda9faeea018afe895 (127.0.93.189:45349)
16:07:43.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.191426 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 896fb523ff8c4a1f8c24b043bbdd1507 in term 1.
16:07:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.192535 558 leader_election.cc:302] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 896fb523ff8c4a1f8c24b043bbdd1507, d8d0a877704d48fda9faeea018afe895; no voters:
16:07:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.193341 618 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [term 1 FOLLOWER]: Leader election won for term 1
16:07:43.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.206445 618 raft_consensus.cc:686] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [term 1 LEADER]: Becoming Leader. State: Replica: 896fb523ff8c4a1f8c24b043bbdd1507, State: Running, Role: LEADER
16:07:43.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.207482 618 consensus_queue.cc:237] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } }
16:07:43.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.218602 645 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
16:07:43.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.232801 642 sys_catalog.cc:455] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 896fb523ff8c4a1f8c24b043bbdd1507. Latest consensus state: current_term: 1 leader_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } } }
16:07:43.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.234352 642 sys_catalog.cc:458] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [sys.catalog]: This master's current role is: LEADER
16:07:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.237810 647 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
16:07:43.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.245649 647 catalog_manager.cc:1484] Initializing Kudu cluster ID...
16:07:43.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.226687 456 master_runner.cc:186] Error getting master registration for 127.0.93.190:43221: OK, instance_id { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" instance_seqno: 1790438858821809 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:07:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.263891 456 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:07:43.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.271876 504 raft_consensus.cc:1261] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895 [term 1 FOLLOWER]: Refusing update from remote peer 896fb523ff8c4a1f8c24b043bbdd1507: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:07:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.273476 642 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:07:43.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.295142 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 896fb523ff8c4a1f8c24b043bbdd1507. Latest consensus state: current_term: 1 leader_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } } }
16:07:43.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.295986 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895 [sys.catalog]: This master's current role is: FOLLOWER
16:07:43.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.315074 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } } }
16:07:43.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.316244 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895 [sys.catalog]: This master's current role is: FOLLOWER
16:07:43.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.319453 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } } }
16:07:43.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.320546 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [sys.catalog]: This master's current role is: LEADER
16:07:43.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.321745 647 catalog_manager.cc:1347] Generated new cluster ID: db795cd1bf8d41948c80340f6d5da1a8
16:07:43.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.322031 647 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
16:07:43.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.366328 647 catalog_manager.cc:1370] Generated new certificate authority record
16:07:43.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.369267 647 catalog_manager.cc:1504] Loading token signing keys...
16:07:43.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.393615 647 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507: Generated new TSK 0
16:07:43.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.395129 647 catalog_manager.cc:1514] Initializing in-progress tserver states...
16:07:43.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.438678 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:43.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:43.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-0/wal
16:07:43.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-0/data
16:07:43.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:07:43.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:07:43.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:07:43.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:43.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:43.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:43.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-0/logs
16:07:43.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-0/data/info.pb
16:07:43.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:07:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:07:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:07:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:07:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:07:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:07:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
16:07:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
16:07:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
16:07:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:07:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:43221,127.0.93.189:45349,127.0.93.188:44501
16:07:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35119
16:07:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:07:43.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.548569 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } attempt: 7
16:07:43.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.580364 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5: Bootstrap starting.
16:07:43.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.590031 454 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5: Neither blocks nor log segments found. Creating new log.
16:07:43.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.593146 454 log.cc:824] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5: Log is configured to *not* fsync() on all Append() calls
16:07:43.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.599565 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5: No bootstrap required, opened a new log
16:07:43.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.651604 454 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } }
16:07:43.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.652508 454 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:07:43.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.652810 454 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0b66f7ad88c04ed98e89a3f1d8314bd5, State: Initialized, Role: FOLLOWER
16:07:43.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.665953 454 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } }
16:07:43.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.670958 454 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5 [sys.catalog]: configured and running, proceeding with master startup.
16:07:43.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.671098 428 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5"
16:07:43.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.671880 428 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5 [term 0 FOLLOWER]: Advancing to term 1
16:07:43.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.673959 429 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" is_pre_election: true
16:07:43.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.684325 428 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 896fb523ff8c4a1f8c24b043bbdd1507 in term 1.
16:07:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.686645 427 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5 [term 1 FOLLOWER]: Refusing update from remote peer 896fb523ff8c4a1f8c24b043bbdd1507: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:07:43.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.695796 642 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:07:43.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.670392 657 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } } }
16:07:43.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.714535 657 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5 [sys.catalog]: This master's current role is: FOLLOWER
16:07:43.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.770625 661 mvcc.cc:205] Tried to move back new op lower bound from 7333637583942184960 to 7333637583709184000. Current Snapshot: MvccSnapshot[applied={T|T < 7333637583942184960}]
16:07:43.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.796207 657 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 896fb523ff8c4a1f8c24b043bbdd1507. Latest consensus state: current_term: 1 leader_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } } }
16:07:43.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.800511 657 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5 [sys.catalog]: This master's current role is: FOLLOWER
16:07:43.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.799206 660 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } } }
16:07:43.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.801376 660 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5 [sys.catalog]: This master's current role is: FOLLOWER
16:07:43.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.819989 674 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
16:07:43.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:43.909605 654 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:07:43.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:43.910188 654 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:07:43.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:43.910830 654 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:07:43.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:43.942898 654 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:07:43.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:43.943897 654 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.129
16:07:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.980919 654 tablet_server_runner.cc:78] Tablet server non-default flags:
16:07:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35119
16:07:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:07:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-0/data
16:07:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-0/wal
16:07:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
16:07:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:07:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-0/data/info.pb
16:07:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
16:07:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:07:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:43221,127.0.93.189:45349,127.0.93.188:44501
16:07:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:07:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:07:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:07:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
16:07:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-0/logs
16:07:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:07:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:07:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:07:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:07:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:07:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:07:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:07:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:07:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:07:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.982517 654 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:07:43.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:43.984546 654 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:07:44.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:44.008834 681 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:44.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:44.208248 642 sys_catalog.cc:455] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0b66f7ad88c04ed98e89a3f1d8314bd5" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43221 } } peers { permanent_uuid: "d8d0a877704d48fda9faeea018afe895" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 45349 } } peers { permanent_uuid: "896fb523ff8c4a1f8c24b043bbdd1507" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44501 } } }
16:07:44.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:44.210489 642 sys_catalog.cc:458] T 00000000000000000000000000000000 P 896fb523ff8c4a1f8c24b043bbdd1507 [sys.catalog]: This master's current role is: LEADER
16:07:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:44.233243 645 catalog_manager.cc:1259] Loaded cluster ID: db795cd1bf8d41948c80340f6d5da1a8
16:07:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:44.234460 645 catalog_manager.cc:1552] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895: loading cluster ID for follower catalog manager: success
16:07:44.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:44.248529 645 catalog_manager.cc:1574] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895: acquiring CA information for follower catalog manager: success
16:07:44.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:44.258229 645 catalog_manager.cc:1602] T 00000000000000000000000000000000 P d8d0a877704d48fda9faeea018afe895: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:07:44.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:44.855647 674 catalog_manager.cc:1259] Loaded cluster ID: db795cd1bf8d41948c80340f6d5da1a8
16:07:44.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:44.860383 674 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5: loading cluster ID for follower catalog manager: success
16:07:44.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:44.884333 674 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5: acquiring CA information for follower catalog manager: success
16:07:44.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:44.939461 674 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 0b66f7ad88c04ed98e89a3f1d8314bd5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:07:45.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:44.009927 683 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:45.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:44.010655 654 server_base.cc:1034] running on GCE node
16:07:45.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:44.009215 680 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:45.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.444093 654 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:07:45.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.447166 654 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:07:45.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.448853 654 hybrid_clock.cc:648] HybridClock initialized: now 1790438865448769 us; error 91 us; skew 500 ppm
16:07:45.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.450237 654 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:07:45.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.459255 654 webserver.cc:460] Webserver started at http://127.0.93.129:36487/ using document root <none> and password file <none>
16:07:45.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.460873 654 fs_manager.cc:359] Metadata directory not provided
16:07:45.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.461503 654 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:07:45.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.462316 654 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:07:45.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.470368 654 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-0/data/instance:
16:07:45.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04b1c6b79e914b61bbf49016f73f3381"
16:07:45.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:45 on dist-test-slave-50w2"
16:07:45.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.471987 654 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-0/wal/instance:
16:07:45.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04b1c6b79e914b61bbf49016f73f3381"
16:07:45.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:45 on dist-test-slave-50w2"
16:07:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.482451 654 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.009s sys 0.000s
16:07:45.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.490949 695 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:07:45.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.492753 654 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.002s sys 0.003s
16:07:45.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.493192 654 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-0/data,/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-0/wal
16:07:45.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04b1c6b79e914b61bbf49016f73f3381"
16:07:45.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:45 on dist-test-slave-50w2"
16:07:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.493665 654 fs_report.cc:383] FS layout report
16:07:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:07:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-0/wal
16:07:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-0/wal
16:07:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-0/data/data
16:07:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:07:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:07:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:07:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:07:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:07:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:07:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:07:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:07:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:07:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:07:45.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:07:45.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.566658 654 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:07:45.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.568542 654 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:07:45.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.569088 654 kserver.cc:163] Server-wide thread pool size limit: 3276
16:07:45.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.573140 654 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
16:07:45.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.579267 654 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:07:45.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.579548 654 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:07:45.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.579821 654 ts_tablet_manager.cc:610] Registered 0 tablets
16:07:45.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.580011 654 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:07:45.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.839083 654 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.129:42307
16:07:45.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.839134 807 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.129:42307 every 8 connection(s)
16:07:45.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.844190 654 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-0/data/info.pb
16:07:45.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.847247 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu as pid 654
16:07:45.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.847704 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-0/wal/instance
16:07:45.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.875265 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:45.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:45.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-1/wal
16:07:45.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-1/data
16:07:45.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:07:45.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:07:45.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:07:45.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:45.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:45.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:45.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-1/logs
16:07:45.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-1/data/info.pb
16:07:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:07:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:07:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:07:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:07:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:07:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:07:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
16:07:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
16:07:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
16:07:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:07:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:43221,127.0.93.189:45349,127.0.93.188:44501
16:07:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35119
16:07:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:07:45.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.935777 808 heartbeater.cc:347] Connected to a master server at 127.0.93.188:44501
16:07:45.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.936362 808 heartbeater.cc:464] Registering TS with master...
16:07:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.937920 808 heartbeater.cc:511] Master 127.0.93.188:44501 requested a full tablet report, sending...
16:07:45.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.941988 574 ts_manager.cc:194] Registered new tserver with Master: 04b1c6b79e914b61bbf49016f73f3381 (127.0.93.129:42307)
16:07:45.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.951000 574 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.129:44601
16:07:45.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.964752 809 heartbeater.cc:347] Connected to a master server at 127.0.93.189:45349
16:07:45.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.965305 809 heartbeater.cc:464] Registering TS with master...
16:07:45.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.966303 809 heartbeater.cc:511] Master 127.0.93.189:45349 requested a full tablet report, sending...
16:07:45.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.970679 494 ts_manager.cc:194] Registered new tserver with Master: 04b1c6b79e914b61bbf49016f73f3381 (127.0.93.129:42307)
16:07:45.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.986157 813 heartbeater.cc:347] Connected to a master server at 127.0.93.190:43221
16:07:45.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.986590 813 heartbeater.cc:464] Registering TS with master...
16:07:45.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.987390 813 heartbeater.cc:511] Master 127.0.93.190:43221 requested a full tablet report, sending...
16:07:45.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:45.991019 419 ts_manager.cc:194] Registered new tserver with Master: 04b1c6b79e914b61bbf49016f73f3381 (127.0.93.129:42307)
16:07:46.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:46.427268 812 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:07:46.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:46.427945 812 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:07:46.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:46.428715 812 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:07:46.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:46.483326 812 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:07:46.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:46.484709 812 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.130
16:07:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:46.546698 812 tablet_server_runner.cc:78] Tablet server non-default flags:
16:07:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35119
16:07:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:07:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-1/data
16:07:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-1/wal
16:07:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
16:07:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:07:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-1/data/info.pb
16:07:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
16:07:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:07:46.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:43221,127.0.93.189:45349,127.0.93.188:44501
16:07:46.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:07:46.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:46.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:07:46.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:07:46.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:46.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
16:07:46.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-1/logs
16:07:46.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:46.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:07:46.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:07:46.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:07:46.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:07:46.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:07:46.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:07:46.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:07:46.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:07:46.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:07:46.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:46.551348 812 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:07:46.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:46.554056 812 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:07:46.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:46.582629 823 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:46.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:46.957252 808 heartbeater.cc:503] Master 127.0.93.188:44501 was elected leader, sending a full tablet report...
16:07:47.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:47.993040 804 debug-util.cc:398] Leaking SignalData structure 0x7b08000afa00 after lost signal to thread 675
16:07:47.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:47.994202 804 debug-util.cc:398] Leaking SignalData structure 0x7b08000bbc40 after lost signal to thread 807
16:07:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:46.583086 824 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:48.150660 812 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.570s user 0.502s sys 0.966s
16:07:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:48.151201 812 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.571s user 0.502s sys 0.967s
16:07:48.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:48.003793 822 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 812
16:07:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.165972 812 server_base.cc:1034] running on GCE node
16:07:48.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:48.168042 830 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:48.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.169698 812 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:07:48.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.172417 812 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:07:48.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.174067 812 hybrid_clock.cc:648] HybridClock initialized: now 1790438868173989 us; error 82 us; skew 500 ppm
16:07:48.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.175163 812 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:07:48.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.188481 812 webserver.cc:460] Webserver started at http://127.0.93.130:44419/ using document root <none> and password file <none>
16:07:48.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.189819 812 fs_manager.cc:359] Metadata directory not provided
16:07:48.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.190130 812 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:07:48.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.190661 812 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:07:48.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.197876 812 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-1/data/instance:
16:07:48.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ad6106f61c143559dc935d4d840aa03"
16:07:48.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:48 on dist-test-slave-50w2"
16:07:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.199467 812 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-1/wal/instance:
16:07:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ad6106f61c143559dc935d4d840aa03"
16:07:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:48 on dist-test-slave-50w2"
16:07:48.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.209766 812 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.010s sys 0.001s
16:07:48.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.218783 835 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:07:48.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.220686 812 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.002s sys 0.002s
16:07:48.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.221136 812 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-1/data,/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-1/wal
16:07:48.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ad6106f61c143559dc935d4d840aa03"
16:07:48.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:48 on dist-test-slave-50w2"
16:07:48.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.221628 812 fs_report.cc:383] FS layout report
16:07:48.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:07:48.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-1/wal
16:07:48.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-1/wal
16:07:48.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-1/data/data
16:07:48.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:07:48.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:07:48.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:07:48.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:07:48.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:07:48.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:07:48.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:07:48.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:07:48.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:07:48.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:07:48.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:07:48.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.284461 812 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:07:48.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.286729 812 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:07:48.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.287367 812 kserver.cc:163] Server-wide thread pool size limit: 3276
16:07:48.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.291389 812 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
16:07:48.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.297821 812 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:07:48.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.298148 812 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:07:48.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.298448 812 ts_tablet_manager.cc:610] Registered 0 tablets
16:07:48.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.298641 812 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:07:48.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.537289 812 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.130:42783
16:07:48.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.541524 812 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-1/data/info.pb
16:07:48.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.543119 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu as pid 812
16:07:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.544221 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-1/wal/instance
16:07:48.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.567142 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:48.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:48.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-2/wal
16:07:48.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-2/data
16:07:48.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-2/logs
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-2/data/info.pb
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:43221,127.0.93.189:45349,127.0.93.188:44501
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35119
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:07:48.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.537853 947 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.130:42783 every 8 connection(s)
16:07:48.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.630287 948 heartbeater.cc:347] Connected to a master server at 127.0.93.188:44501
16:07:48.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.630890 948 heartbeater.cc:464] Registering TS with master...
16:07:48.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.632735 948 heartbeater.cc:511] Master 127.0.93.188:44501 requested a full tablet report, sending...
16:07:48.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.635809 574 ts_manager.cc:194] Registered new tserver with Master: 5ad6106f61c143559dc935d4d840aa03 (127.0.93.130:42783)
16:07:48.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.638100 574 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.130:36583
16:07:48.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.659302 952 heartbeater.cc:347] Connected to a master server at 127.0.93.190:43221
16:07:48.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.659726 952 heartbeater.cc:464] Registering TS with master...
16:07:48.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.660471 952 heartbeater.cc:511] Master 127.0.93.190:43221 requested a full tablet report, sending...
16:07:48.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.662865 951 heartbeater.cc:347] Connected to a master server at 127.0.93.189:45349
16:07:48.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.663249 951 heartbeater.cc:464] Registering TS with master...
16:07:48.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.663345 419 ts_manager.cc:194] Registered new tserver with Master: 5ad6106f61c143559dc935d4d840aa03 (127.0.93.130:42783)
16:07:48.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.664037 951 heartbeater.cc:511] Master 127.0.93.189:45349 requested a full tablet report, sending...
16:07:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:48.667623 494 ts_manager.cc:194] Registered new tserver with Master: 5ad6106f61c143559dc935d4d840aa03 (127.0.93.130:42783)
16:07:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:48.983788 958 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:07:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:48.984349 958 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:07:48.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:48.984963 958 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:07:49.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:49.017350 958 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:07:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:49.018606 958 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.131
16:07:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:49.061967 958 tablet_server_runner.cc:78] Tablet server non-default flags:
16:07:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35119
16:07:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:07:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-2/data
16:07:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-2/wal
16:07:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
16:07:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:07:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-2/data/info.pb
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:43221,127.0.93.189:45349,127.0.93.188:44501
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-2/logs
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:49.063531 958 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:07:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:49.065320 958 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:07:49.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:49.089848 968 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:49.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:49.644953 948 heartbeater.cc:503] Master 127.0.93.188:44501 was elected leader, sending a full tablet report...
16:07:50.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:49.091584 970 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:50.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:49.093104 958 server_base.cc:1034] running on GCE node
16:07:50.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:49.092624 967 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:50.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.341190 958 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:07:50.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.344101 958 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:07:50.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.345611 958 hybrid_clock.cc:648] HybridClock initialized: now 1790438870345554 us; error 74 us; skew 500 ppm
16:07:50.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.346380 958 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:07:50.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.352936 958 webserver.cc:460] Webserver started at http://127.0.93.131:37393/ using document root <none> and password file <none>
16:07:50.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.353924 958 fs_manager.cc:359] Metadata directory not provided
16:07:50.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.354118 958 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:07:50.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.354481 958 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:07:50.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.359421 958 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-2/data/instance:
16:07:50.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f52c1c105421468d9fa2002f3b3ad3c3"
16:07:50.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:50 on dist-test-slave-50w2"
16:07:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.360647 958 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-2/wal/instance:
16:07:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f52c1c105421468d9fa2002f3b3ad3c3"
16:07:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:50 on dist-test-slave-50w2"
16:07:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.368028 958 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
16:07:50.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.374172 978 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:07:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.375559 958 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
16:07:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.375834 958 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-2/data,/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-2/wal
16:07:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f52c1c105421468d9fa2002f3b3ad3c3"
16:07:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:50 on dist-test-slave-50w2"
16:07:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.376159 958 fs_report.cc:383] FS layout report
16:07:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:07:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-2/wal
16:07:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-2/wal
16:07:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-2/data/data
16:07:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:07:50.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:07:50.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:07:50.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:07:50.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:07:50.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:07:50.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:07:50.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:07:50.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:07:50.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:07:50.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:07:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.429713 958 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:07:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.431141 958 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:07:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.431569 958 kserver.cc:163] Server-wide thread pool size limit: 3276
16:07:50.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.434031 958 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
16:07:50.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.438393 958 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:07:50.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.438609 958 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:07:50.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.438822 958 ts_tablet_manager.cc:610] Registered 0 tablets
16:07:50.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.438961 958 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:07:50.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.608462 958 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.131:43795
16:07:50.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.608572 1090 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.131:43795 every 8 connection(s)
16:07:50.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.611333 958 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-2/data/info.pb
16:07:50.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.619148 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu as pid 958
16:07:50.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.619766 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5018394485381103871/ts-2/wal/instance
16:07:50.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.673278 1091 heartbeater.cc:347] Connected to a master server at 127.0.93.188:44501
16:07:50.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.673781 1091 heartbeater.cc:464] Registering TS with master...
16:07:50.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.674976 1091 heartbeater.cc:511] Master 127.0.93.188:44501 requested a full tablet report, sending...
16:07:50.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.677461 574 ts_manager.cc:194] Registered new tserver with Master: f52c1c105421468d9fa2002f3b3ad3c3 (127.0.93.131:43795)
16:07:50.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.679899 574 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.131:51097
16:07:50.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.683828 1092 heartbeater.cc:347] Connected to a master server at 127.0.93.189:45349
16:07:50.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.684149 1092 heartbeater.cc:464] Registering TS with master...
16:07:50.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.684890 1092 heartbeater.cc:511] Master 127.0.93.189:45349 requested a full tablet report, sending...
16:07:50.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.687801 494 ts_manager.cc:194] Registered new tserver with Master: f52c1c105421468d9fa2002f3b3ad3c3 (127.0.93.131:43795)
16:07:50.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.694098 1093 heartbeater.cc:347] Connected to a master server at 127.0.93.190:43221
16:07:50.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.694557 1093 heartbeater.cc:464] Registering TS with master...
16:07:50.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.695781 1093 heartbeater.cc:511] Master 127.0.93.190:43221 requested a full tablet report, sending...
16:07:50.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.697953 418 ts_manager.cc:194] Registered new tserver with Master: f52c1c105421468d9fa2002f3b3ad3c3 (127.0.93.131:43795)
16:07:50.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:50.701378 374 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
16:07:50.712 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:07:50.714 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
16:07:50.725 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.93.190"
port: 43221
}
bound_http_address {
host: "127.0.93.190"
port: 44191
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.93.189"
port: 45349
}
bound_http_address {
host: "127.0.93.189"
port: 42845
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.93.188"
port: 44501
}
bound_http_address {
host: "127.0.93.188"
port: 41759
}
}
}
16:07:50.731 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:07:50.734 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.93.129"
port: 42307
}
bound_http_address {
host: "127.0.93.129"
port: 36487
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.93.130"
port: 42783
}
bound_http_address {
host: "127.0.93.130"
port: 44419
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.93.131"
port: 43795
}
bound_http_address {
host: "127.0.93.131"
port: 37393
}
}
}
16:07:50.734 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
16:07:50.918 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
16:07:51.043 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskmIbLz2/test-tmp
16:07:51.044 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
16:07:51.044 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskmIbLz2/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
16:07:51.049 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
16:07:51.396 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:07:51.396 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:07:51.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:51.399962 1103 mini_oidc.cc:152] Starting JWKS server
16:07:51.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:51.404106 1103 webserver.cc:460] Webserver started at http://127.0.0.1:38015/ using document root <none> and password file <none>
16:07:51.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:51.404879 1103 mini_oidc.cc:189] Starting OIDC Discovery server
16:07:51.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:51.406929 1103 webserver.cc:460] Webserver started at http://127.0.0.1:41121/ using document root <none> and password file <none>
16:07:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
16:07:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/principal' for realm 'KRBTEST.COM',
16:07:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
16:07:51.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:51 dist-test-slave-50w2 krb5kdc[1115](info): setting up network...
16:07:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
16:07:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:51 dist-test-slave-50w2 krb5kdc[1115](info): set up 2 sockets
16:07:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:51 dist-test-slave-50w2 krb5kdc[1115](info): commencing operation
16:07:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
16:07:51.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:51.685389 1091 heartbeater.cc:503] Master 127.0.93.188:44501 was elected leader, sending a full tablet report...
16:07:52.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:52.545958 1103 mini_kdc.cc:121] Time spent starting KDC: real 1.113s user 0.001s sys 0.023s
16:07:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
16:07:52.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:07:52.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
16:07:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
16:07:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:07:52.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
16:07:52.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
16:07:52.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:07:52.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
16:07:52.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:07:52.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/test-user.keytab.
16:07:52.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/test-user.keytab.
16:07:52.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:52 dist-test-slave-50w2 krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438872, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:07:52.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:07:52.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T16:07:52Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
16:07:52.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T16:07:52Z Disabled control of system clock
16:07:52.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.254@KRBTEST.COM; defaulting to no policy
16:07:52.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:07:52.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.254@KRBTEST.COM" created.
16:07:52.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:07:52.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kudu_127.1.19.254.keytab.
16:07:52.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kudu_127.1.19.254.keytab.
16:07:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:52.779088 1103 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/wal
16:07:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/data
16:07:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:07:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:07:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:07:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/logs
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/data/info.pb
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.254:45317
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.254
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kudu_127.1.19.254.keytab
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.254
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:43623
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:07:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:07:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45317,127.1.19.253:45027,127.1.19.252:41153
16:07:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:07:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38015/jwks.json
16:07:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:07:53.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:53.131875 1128 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:07:53.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:53.133052 1128 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:07:53.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:53.134050 1128 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:07:53.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:53.165745 1128 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:07:53.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:53.165989 1128 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:07:53.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:53.166244 1128 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:07:53.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:53.166520 1128 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:07:53.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:53.166838 1128 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:07:53.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:53.167161 1128 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:07:53.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:53.167407 1128 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38015/jwks.json
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:53.208691 1128 master_runner.cc:386] Master server non-default flags:
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:43623
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/data
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/wal
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45317,127.1.19.253:45027,127.1.19.252:41153
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kudu_127.1.19.254.keytab
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.254
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.254:45317
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38015/jwks.json
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/data/info.pb
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.254
16:07:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:07:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:07:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:07:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:07:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/logs
16:07:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:07:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:07:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:07:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:07:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:07:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:07:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:07:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:07:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:07:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:53.209889 1128 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:07:53.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:53.210846 1128 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:07:53.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:53.231222 1137 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:53.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:53.929980 804 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9d00 after lost signal to thread 675
16:07:53.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:53.932330 804 debug-util.cc:398] Leaking SignalData structure 0x7b08000afb20 after lost signal to thread 807
16:07:54.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:53.231604 1135 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:54.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:53.233583 1128 server_base.cc:1034] running on GCE node
16:07:54.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:53.232272 1134 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:54.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.512037 1128 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:07:54.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.515089 1128 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:07:54.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.516593 1128 hybrid_clock.cc:648] HybridClock initialized: now 1790438874516559 us; error 43 us; skew 500 ppm
16:07:54.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.517660 1128 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:07:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:54 dist-test-slave-50w2 krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438874, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:07:54.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.531701 1128 init.cc:377] Logged in from keytab as kudu/127.1.19.254@KRBTEST.COM (short username kudu)
16:07:54.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.538679 1128 webserver.cc:460] Webserver started at http://127.1.19.254:42345/ using document root <none> and password file <none>
16:07:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.539875 1128 fs_manager.cc:359] Metadata directory not provided
16:07:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.540088 1128 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:07:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.540503 1128 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:07:54.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.546137 1128 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/data/instance:
16:07:54.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de38a497494a4fe9b7efbda09e530ab3"
16:07:54.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:54 on dist-test-slave-50w2"
16:07:54.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.547411 1128 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/wal/instance:
16:07:54.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de38a497494a4fe9b7efbda09e530ab3"
16:07:54.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:54 on dist-test-slave-50w2"
16:07:54.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.555411 1128 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.000s sys 0.008s
16:07:54.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.562114 1146 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:07:54.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.563488 1128 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.000s sys 0.004s
16:07:54.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.563809 1128 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/data,/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/wal
16:07:54.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de38a497494a4fe9b7efbda09e530ab3"
16:07:54.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:54 on dist-test-slave-50w2"
16:07:54.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.564153 1128 fs_report.cc:383] FS layout report
16:07:54.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:07:54.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/wal
16:07:54.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/wal
16:07:54.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/data/data
16:07:54.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:07:54.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:07:54.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:07:54.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:07:54.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:07:54.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:07:54.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:07:54.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:07:54.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:07:54.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:07:54.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:07:54.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.599680 1128 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:07:54.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.606315 1128 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:07:54.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.606920 1128 kserver.cc:163] Server-wide thread pool size limit: 3276
16:07:54.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.704948 1128 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.254:45317
16:07:54.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.705127 1197 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.254:45317 every 8 connection(s)
16:07:54.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.708647 1128 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/data/info.pb
16:07:54.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.714108 1103 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu as pid 1128
16:07:54.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.714771 1103 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/wal/instance
16:07:54.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.715960 1198 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:07:54.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.733924 1198 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45317 } has no permanent_uuid. Determining permanent_uuid...
16:07:54.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.253@KRBTEST.COM; defaulting to no policy
16:07:54.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:07:54.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.253@KRBTEST.COM" created.
16:07:54.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:54 dist-test-slave-50w2 krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438874, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.254@KRBTEST.COM for kudu/127.1.19.254@KRBTEST.COM
16:07:54.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:07:54.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kudu_127.1.19.253.keytab.
16:07:54.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kudu_127.1.19.253.keytab.
16:07:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.774843 1103 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/wal
16:07:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/data
16:07:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:07:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:07:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:07:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/logs
16:07:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/data/info.pb
16:07:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:07:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:07:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.253:45027
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.253
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kudu_127.1.19.253.keytab
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.253
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:43623
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:07:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45317,127.1.19.253:45027,127.1.19.252:41153
16:07:54.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:07:54.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38015/jwks.json
16:07:54.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:07:54.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.780563 1198 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } has no permanent_uuid. Determining permanent_uuid...
16:07:54.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:54.783329 1149 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.253:45027: connect: Connection refused (error 111)
16:07:54.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:54.788057 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.253:45027: Network error: Client connection negotiation failed: client connection to 127.1.19.253:45027: connect: Connection refused (error 111)
16:07:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.837551 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } attempt: 1
16:07:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:54.841530 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.253:45027: Network error: Client connection negotiation failed: client connection to 127.1.19.253:45027: connect: Connection refused (error 111)
16:07:54.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:54.909961 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } attempt: 2
16:07:54.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:54.914631 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.253:45027: Network error: Client connection negotiation failed: client connection to 127.1.19.253:45027: connect: Connection refused (error 111)
16:07:55.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:55.006028 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } attempt: 3
16:07:55.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:55.010320 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.253:45027: Network error: Client connection negotiation failed: client connection to 127.1.19.253:45027: connect: Connection refused (error 111)
16:07:55.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:55.108204 1204 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:07:55.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:55.108731 1204 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:07:55.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:55.109321 1204 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:07:55.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:55.141292 1204 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:07:55.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:55.141567 1204 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:07:55.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:55.141940 1204 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:07:55.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:55.142153 1204 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:07:55.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:55.142339 1204 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:07:55.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:55.142530 1204 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:07:55.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:55.142670 1204 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38015/jwks.json
16:07:55.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:55.153739 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } attempt: 4
16:07:55.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:55.158565 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.253:45027: Network error: Client connection negotiation failed: client connection to 127.1.19.253:45027: connect: Connection refused (error 111)
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:55.181921 1204 master_runner.cc:386] Master server non-default flags:
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:43623
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/data
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/wal
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45317,127.1.19.253:45027,127.1.19.252:41153
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kudu_127.1.19.253.keytab
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.253
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.253:45027
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38015/jwks.json
16:07:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/data/info.pb
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.253
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/logs
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:07:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:07:55.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:55.183103 1204 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:07:55.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:55.184098 1204 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:07:55.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:55.203131 1214 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:55.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:55.458001 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } attempt: 5
16:07:55.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:55.462244 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.253:45027: Network error: Client connection negotiation failed: client connection to 127.1.19.253:45027: connect: Connection refused (error 111)
16:07:56.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.009615 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } attempt: 6
16:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:56.019986 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.253:45027: Network error: Client connection negotiation failed: client connection to 127.1.19.253:45027: connect: Connection refused (error 111)
16:07:56.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:55.203123 1211 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:56.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:55.203213 1212 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:56.448407 1213 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
16:07:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.448613 1204 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:07:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.454290 1204 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:07:56.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.456900 1204 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:07:56.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.458320 1204 hybrid_clock.cc:648] HybridClock initialized: now 1790438876458297 us; error 70 us; skew 500 ppm
16:07:56.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.459336 1204 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:07:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:56 dist-test-slave-50w2 krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438876, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:07:56.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.466158 1204 init.cc:377] Logged in from keytab as kudu/127.1.19.253@KRBTEST.COM (short username kudu)
16:07:56.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.470999 1204 webserver.cc:460] Webserver started at http://127.1.19.253:37601/ using document root <none> and password file <none>
16:07:56.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.472121 1204 fs_manager.cc:359] Metadata directory not provided
16:07:56.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.472409 1204 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:07:56.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.472991 1204 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:07:56.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.478708 1204 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/data/instance:
16:07:56.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8d0b5ea683574cf4bf37f74a0010487e"
16:07:56.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:56 on dist-test-slave-50w2"
16:07:56.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.479758 1204 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/wal/instance:
16:07:56.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8d0b5ea683574cf4bf37f74a0010487e"
16:07:56.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:56 on dist-test-slave-50w2"
16:07:56.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.487828 1204 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
16:07:56.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.494309 1226 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:07:56.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.495793 1204 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
16:07:56.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.496136 1204 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/data,/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/wal
16:07:56.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8d0b5ea683574cf4bf37f74a0010487e"
16:07:56.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:56 on dist-test-slave-50w2"
16:07:56.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.496454 1204 fs_report.cc:383] FS layout report
16:07:56.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:07:56.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/wal
16:07:56.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/wal
16:07:56.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/data/data
16:07:56.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:07:56.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:07:56.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:07:56.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:07:56.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:07:56.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:07:56.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:07:56.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:07:56.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:07:56.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:07:56.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:07:56.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.535566 1204 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:07:56.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.542253 1204 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:07:56.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.542797 1204 kserver.cc:163] Server-wide thread pool size limit: 3276
16:07:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.630524 1204 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.253:45027
16:07:56.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.630589 1277 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.253:45027 every 8 connection(s)
16:07:56.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.633319 1204 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/data/info.pb
16:07:56.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.638054 1103 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu as pid 1204
16:07:56.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.638597 1103 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/wal/instance
16:07:56.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.641402 1278 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:07:56.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.657053 1278 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45317 } has no permanent_uuid. Determining permanent_uuid...
16:07:56.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.252@KRBTEST.COM; defaulting to no policy
16:07:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:07:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.252@KRBTEST.COM" created.
16:07:56.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:56 dist-test-slave-50w2 krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438876, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.253@KRBTEST.COM for kudu/127.1.19.254@KRBTEST.COM
16:07:56.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:07:56.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kudu_127.1.19.252.keytab.
16:07:56.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kudu_127.1.19.252.keytab.
16:07:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.700359 1103 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/wal
16:07:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/data
16:07:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:07:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:07:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:07:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/logs
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/data/info.pb
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.252:41153
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.252
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kudu_127.1.19.252.keytab
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.252
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:07:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:07:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:43623
16:07:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:07:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:07:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45317,127.1.19.253:45027,127.1.19.252:41153
16:07:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:07:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38015/jwks.json
16:07:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:07:56.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.710677 1278 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } has no permanent_uuid. Determining permanent_uuid...
16:07:56.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:56 dist-test-slave-50w2 krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438876, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.253@KRBTEST.COM for kudu/127.1.19.253@KRBTEST.COM
16:07:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.755177 1278 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } has no permanent_uuid. Determining permanent_uuid...
16:07:56.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:56.757423 1229 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.252:41153: connect: Connection refused (error 111)
16:07:56.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:56.762172 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:41153: Network error: Client connection negotiation failed: client connection to 127.1.19.252:41153: connect: Connection refused (error 111)
16:07:56.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.811726 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } attempt: 1
16:07:56.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:56.815773 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:41153: Network error: Client connection negotiation failed: client connection to 127.1.19.252:41153: connect: Connection refused (error 111)
16:07:56.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.884248 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } attempt: 2
16:07:56.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:56.888342 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:41153: Network error: Client connection negotiation failed: client connection to 127.1.19.252:41153: connect: Connection refused (error 111)
16:07:56.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:56.979873 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } attempt: 3
16:07:56.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:56.984582 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:41153: Network error: Client connection negotiation failed: client connection to 127.1.19.252:41153: connect: Connection refused (error 111)
16:07:57.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.070132 1284 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:07:57.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.070734 1284 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:07:57.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.071322 1284 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:07:57.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:57.080926 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } attempt: 7
16:07:57.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:57 dist-test-slave-50w2 krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438874, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.254@KRBTEST.COM for kudu/127.1.19.253@KRBTEST.COM
16:07:57.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:57.113731 1198 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } has no permanent_uuid. Determining permanent_uuid...
16:07:57.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.113938 1284 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:07:57.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.114204 1284 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:07:57.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.114655 1284 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:07:57.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.115026 1284 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:07:57.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.115367 1284 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:07:57.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.115847 1284 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:07:57.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.116107 1284 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38015/jwks.json
16:07:57.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.119858 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:41153: Network error: Client connection negotiation failed: client connection to 127.1.19.252:41153: connect: Connection refused (error 111)
16:07:57.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:57.128082 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } attempt: 4
16:07:57.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.132300 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:41153: Network error: Client connection negotiation failed: client connection to 127.1.19.252:41153: connect: Connection refused (error 111)
16:07:57.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:57.155757 1284 master_runner.cc:386] Master server non-default flags:
16:07:57.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:57.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:43623
16:07:57.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:57.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:07:57.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/data
16:07:57.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/wal
16:07:57.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:07:57.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:07:57.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45317,127.1.19.253:45027,127.1.19.252:41153
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kudu_127.1.19.252.keytab
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.252
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.252:41153
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38015/jwks.json
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/data/info.pb
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.252
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/logs
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:07:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:57.156929 1284 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:07:57.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:57.158171 1284 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:07:57.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:57.178227 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } attempt: 1
16:07:57.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.186471 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:41153: Network error: Client connection negotiation failed: client connection to 127.1.19.252:41153: connect: Connection refused (error 111)
16:07:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:57.268069 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } attempt: 2
16:07:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.275187 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:41153: Network error: Client connection negotiation failed: client connection to 127.1.19.252:41153: connect: Connection refused (error 111)
16:07:57.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:57.360790 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } attempt: 3
16:07:57.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.367031 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:41153: Network error: Client connection negotiation failed: client connection to 127.1.19.252:41153: connect: Connection refused (error 111)
16:07:57.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:57.431769 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } attempt: 5
16:07:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.439178 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:41153: Network error: Client connection negotiation failed: client connection to 127.1.19.252:41153: connect: Connection refused (error 111)
16:07:57.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:57.507597 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } attempt: 4
16:07:57.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.514497 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:41153: Network error: Client connection negotiation failed: client connection to 127.1.19.252:41153: connect: Connection refused (error 111)
16:07:57.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:57.798199 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } attempt: 5
16:07:57.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.804777 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:41153: Network error: Client connection negotiation failed: client connection to 127.1.19.252:41153: connect: Connection refused (error 111)
16:07:57.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:57.986770 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } attempt: 6
16:07:57.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.997223 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:41153: Network error: Client connection negotiation failed: client connection to 127.1.19.252:41153: connect: Connection refused (error 111)
16:07:58.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.357402 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } attempt: 6
16:07:58.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:58.366464 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:41153: Network error: Client connection negotiation failed: client connection to 127.1.19.252:41153: connect: Connection refused (error 111)
16:07:58.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.179936 1295 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.181092 1296 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:57.179893 1298 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:07:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:58.520174 1297 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1342 milliseconds
16:07:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.520203 1284 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:07:58.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.525362 1284 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:07:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.528285 1284 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:07:58.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.529652 1284 hybrid_clock.cc:648] HybridClock initialized: now 1790438878529631 us; error 66 us; skew 500 ppm
16:07:58.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.530467 1284 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:07:58.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:58 dist-test-slave-50w2 krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438878, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:07:58.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.537750 1284 init.cc:377] Logged in from keytab as kudu/127.1.19.252@KRBTEST.COM (short username kudu)
16:07:58.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.546670 1284 webserver.cc:460] Webserver started at http://127.1.19.252:42087/ using document root <none> and password file <none>
16:07:58.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.547886 1284 fs_manager.cc:359] Metadata directory not provided
16:07:58.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.548170 1284 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:07:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.548794 1284 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:07:58.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.554392 1284 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/data/instance:
16:07:58.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca4d7764cafd4e12a168e1ccaa7ae84c"
16:07:58.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:58 on dist-test-slave-50w2"
16:07:58.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.555855 1284 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/wal/instance:
16:07:58.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca4d7764cafd4e12a168e1ccaa7ae84c"
16:07:58.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:58 on dist-test-slave-50w2"
16:07:58.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.564723 1284 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
16:07:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.571935 1313 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:07:58.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.573791 1284 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
16:07:58.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.574116 1284 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/data,/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/wal
16:07:58.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca4d7764cafd4e12a168e1ccaa7ae84c"
16:07:58.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:58 on dist-test-slave-50w2"
16:07:58.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.574425 1284 fs_report.cc:383] FS layout report
16:07:58.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:07:58.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/wal
16:07:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/wal
16:07:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/data/data
16:07:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:07:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:07:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:07:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:07:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:07:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:07:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:07:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:07:58.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:07:58.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:07:58.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:07:58.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.664750 1284 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:07:58.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.671330 1284 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:07:58.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.671844 1284 kserver.cc:163] Server-wide thread pool size limit: 3276
16:07:58.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.772635 1284 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.252:41153
16:07:58.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.772738 1364 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.252:41153 every 8 connection(s)
16:07:58.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.775444 1284 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/data/info.pb
16:07:58.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.782215 1365 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:07:58.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.783319 1103 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu as pid 1284
16:07:58.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.783746 1103 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/wal/instance
16:07:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.801172 1365 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45317 } has no permanent_uuid. Determining permanent_uuid...
16:07:58.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:58 dist-test-slave-50w2 krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438878, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.252@KRBTEST.COM for kudu/127.1.19.254@KRBTEST.COM
16:07:58.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:58 dist-test-slave-50w2 krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438872, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.252@KRBTEST.COM
16:07:58.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.842183 1365 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } has no permanent_uuid. Determining permanent_uuid...
16:07:58.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:58 dist-test-slave-50w2 krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438878, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.252@KRBTEST.COM for kudu/127.1.19.253@KRBTEST.COM
16:07:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:58 dist-test-slave-50w2 krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438872, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.253@KRBTEST.COM
16:07:58.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.878052 1365 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } has no permanent_uuid. Determining permanent_uuid...
16:07:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:58 dist-test-slave-50w2 krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438878, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.252@KRBTEST.COM for kudu/127.1.19.252@KRBTEST.COM
16:07:58.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:58 dist-test-slave-50w2 krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438872, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.254@KRBTEST.COM
16:07:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.912585 1365 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: Bootstrap starting.
16:07:58.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.914697 1103 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
16:07:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.919440 1365 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: Neither blocks nor log segments found. Creating new log.
16:07:58.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.921238 1365 log.cc:824] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: Log is configured to *not* fsync() on all Append() calls
16:07:58.922 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:07:58.923 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
16:07:58.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.926769 1365 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: No bootstrap required, opened a new log
16:07:58.934 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
16:07:58.935 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
16:07:58.938 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.19.254"
port: 45317
}
bound_http_address {
host: "127.1.19.254"
port: 42345
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.19.253"
port: 45027
}
bound_http_address {
host: "127.1.19.253"
port: 37601
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.19.252"
port: 41153
}
bound_http_address {
host: "127.1.19.252"
port: 42087
}
}
}
16:07:58.939 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:07:58.940 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:07:58.940 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:07:58.942 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:07:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.954231 1365 raft_consensus.cc:348] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de38a497494a4fe9b7efbda09e530ab3" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45317 } } peers { permanent_uuid: "8d0b5ea683574cf4bf37f74a0010487e" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } } peers { permanent_uuid: "ca4d7764cafd4e12a168e1ccaa7ae84c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } }
16:07:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.955116 1365 raft_consensus.cc:374] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:07:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.955408 1365 raft_consensus.cc:729] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ca4d7764cafd4e12a168e1ccaa7ae84c, State: Initialized, Role: FOLLOWER
16:07:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.956387 1365 consensus_queue.cc:260] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de38a497494a4fe9b7efbda09e530ab3" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45317 } } peers { permanent_uuid: "8d0b5ea683574cf4bf37f74a0010487e" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } } peers { permanent_uuid: "ca4d7764cafd4e12a168e1ccaa7ae84c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } }
16:07:58.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.960065 1375 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de38a497494a4fe9b7efbda09e530ab3" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45317 } } peers { permanent_uuid: "8d0b5ea683574cf4bf37f74a0010487e" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } } peers { permanent_uuid: "ca4d7764cafd4e12a168e1ccaa7ae84c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } } }
16:07:58.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.961403 1375 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [sys.catalog]: This master's current role is: FOLLOWER
16:07:58.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.963212 1365 sys_catalog.cc:564] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [sys.catalog]: configured and running, proceeding with master startup.
16:07:58.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:58 dist-test-slave-50w2 krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438878, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:07:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:07:58.996 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:07:58.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:58.997428 1388 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:07:58.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:58.997836 1388 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:07:58.999 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/krb5cc
16:07:59.006 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:07:59.007 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/krb5cc
16:07:59.008 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:07:59.013 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
16:07:59.015 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:07:59.016 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:07:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:58.991392 1284 master_runner.cc:186] Error getting master registration for 127.1.19.254:45317: OK, instance_id { permanent_uuid: "de38a497494a4fe9b7efbda09e530ab3" instance_seqno: 1790438874678654 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:07:59.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:59.027180 1284 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:07:59.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:07:59 dist-test-slave-50w2 krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438879, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:07:59.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:07:59.052 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:07:59.052 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/krb5cc
16:07:59.054 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:07:59.054 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.252:41153
16:07:59.056 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
16:07:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:59.057436 1103 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu with pid 1284
16:07:59.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:59.057806 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } attempt: 7
16:07:59.087 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:07:59.088 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.254:45317
16:07:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:59.087956 1392 negotiation.cc:337] Failed RPC negotiation. Trace:
16:07:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0926 16:07:59.059030 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.1.19.252:41153 (local address 127.0.0.1:33798)
16:07:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0926 16:07:59.071564 (+ 12534us) negotiation.cc:107] Waiting for socket to connect
16:07:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0926 16:07:59.071621 (+ 57us) client_negotiation.cc:174] Beginning negotiation
16:07:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0926 16:07:59.071906 (+ 285us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
16:07:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0926 16:07:59.086479 (+ 14573us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.1.19.252:41153: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
16:07:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":11979,"thread_start_us":12975,"threads_started":1}
16:07:59.088 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
16:07:59.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:59.091042 1103 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu with pid 1128
16:07:59.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:59.095670 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:41153: Network error: Client connection negotiation failed: client connection to 127.1.19.252:41153: connect: Connection refused (error 111)
16:07:59.115 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:07:59.115 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.253:45027
16:07:59.115 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
16:07:59.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:59.116175 1103 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu with pid 1204
16:07:59.138 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:07:59.138 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.252:41153
16:07:59.140 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
16:07:59.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:59.141428 1103 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:59.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/wal
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/data
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/logs
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/data/info.pb
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:07:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.252:41153
16:07:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.252
16:07:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42087
16:07:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kudu_127.1.19.252.keytab
16:07:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.252
16:07:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:07:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:07:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:07:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:43623
16:07:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:07:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:07:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45317,127.1.19.253:45027,127.1.19.252:41153
16:07:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:07:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38015/jwks.json
16:07:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:07:59.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:59.457523 1394 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:07:59.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:59.458071 1394 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:07:59.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:59.458644 1394 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:07:59.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:59.491355 1394 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:07:59.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:59.491596 1394 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:07:59.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:59.491856 1394 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:07:59.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:59.492060 1394 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:07:59.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:59.492321 1394 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:07:59.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:59.492518 1394 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:07:59.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:59.492658 1394 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38015/jwks.json
16:07:59.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:59.531229 1394 master_runner.cc:386] Master server non-default flags:
16:07:59.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:07:59.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:43623
16:07:59.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:07:59.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:07:59.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/data
16:07:59.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/wal
16:07:59.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:07:59.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:07:59.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45317,127.1.19.253:45027,127.1.19.252:41153
16:07:59.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kudu_127.1.19.252.keytab
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.252
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.252:41153
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38015/jwks.json
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/data/info.pb
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.252
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42087
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/logs
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:07:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:59.532351 1394 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:07:59.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:59.533316 1394 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:07:59.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:59.552510 1402 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:08:00.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:59.553329 1404 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:08:00.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:07:59.554523 1394 server_base.cc:1034] running on GCE node
16:08:00.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:07:59.552531 1401 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:08:00.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:00.854760 1394 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:08:00.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:00.858430 1394 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:08:00.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:00.860214 1394 hybrid_clock.cc:648] HybridClock initialized: now 1790438880860160 us; error 46 us; skew 500 ppm
16:08:00.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:00.861277 1394 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:08:00.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:08:00 dist-test-slave-50w2 krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438880, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:08:00.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:00.874256 1394 init.cc:377] Logged in from keytab as kudu/127.1.19.252@KRBTEST.COM (short username kudu)
16:08:00.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:00.879832 1394 webserver.cc:460] Webserver started at http://127.1.19.252:42087/ using document root <none> and password file <none>
16:08:00.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:00.880759 1394 fs_manager.cc:359] Metadata directory not provided
16:08:00.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:00.880964 1394 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:08:00.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:00.889195 1394 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
16:08:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:00.894716 1412 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:08:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:00.896127 1394 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
16:08:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:00.896539 1394 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/data,/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/wal
16:08:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca4d7764cafd4e12a168e1ccaa7ae84c"
16:08:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:58 on dist-test-slave-50w2"
16:08:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:00.899549 1394 fs_report.cc:383] FS layout report
16:08:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:08:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/wal
16:08:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/wal
16:08:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/data/data
16:08:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:08:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:08:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:08:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:08:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:08:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:08:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:08:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:08:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:08:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:08:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:08:00.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:00.946384 1394 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:08:00.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:00.953372 1394 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:08:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:00.954033 1394 kserver.cc:163] Server-wide thread pool size limit: 3276
16:08:01.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.030360 1394 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.252:41153
16:08:01.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.030567 1463 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.252:41153 every 8 connection(s)
16:08:01.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.033222 1394 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-2/data/info.pb
16:08:01.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.042387 1103 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu as pid 1394
16:08:01.043 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:08:01.043 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.254:45317
16:08:01.043 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
16:08:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.045185 1103 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:08:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:08:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/wal
16:08:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/data
16:08:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:08:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:08:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:08:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:08:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:08:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:08:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/logs
16:08:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/data/info.pb
16:08:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:08:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:08:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:08:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:08:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.254:45317
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.254
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42345
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kudu_127.1.19.254.keytab
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.254
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:43623
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45317,127.1.19.253:45027,127.1.19.252:41153
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38015/jwks.json
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.045559 1464 sys_catalog.cc:263] Verifying existing consensus state
16:08:01.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.055222 1464 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: Bootstrap starting.
16:08:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.099557 1464 log.cc:824] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: Log is configured to *not* fsync() on all Append() calls
16:08:01.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.105624 1464 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:08:01.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.106649 1464 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: Bootstrap complete.
16:08:01.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.135028 1464 raft_consensus.cc:348] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de38a497494a4fe9b7efbda09e530ab3" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45317 } } peers { permanent_uuid: "8d0b5ea683574cf4bf37f74a0010487e" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } } peers { permanent_uuid: "ca4d7764cafd4e12a168e1ccaa7ae84c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } }
16:08:01.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.137615 1464 raft_consensus.cc:374] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:08:01.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.138095 1464 raft_consensus.cc:729] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ca4d7764cafd4e12a168e1ccaa7ae84c, State: Initialized, Role: FOLLOWER
16:08:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.139128 1464 consensus_queue.cc:260] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de38a497494a4fe9b7efbda09e530ab3" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45317 } } peers { permanent_uuid: "8d0b5ea683574cf4bf37f74a0010487e" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } } peers { permanent_uuid: "ca4d7764cafd4e12a168e1ccaa7ae84c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } }
16:08:01.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.142010 1469 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de38a497494a4fe9b7efbda09e530ab3" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45317 } } peers { permanent_uuid: "8d0b5ea683574cf4bf37f74a0010487e" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } } peers { permanent_uuid: "ca4d7764cafd4e12a168e1ccaa7ae84c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } } }
16:08:01.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.143139 1464 sys_catalog.cc:564] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [sys.catalog]: configured and running, proceeding with master startup.
16:08:01.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.143131 1469 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [sys.catalog]: This master's current role is: FOLLOWER
16:08:01.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.174547 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:01.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.175179 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.171387 1394 master_runner.cc:186] Error getting master registration for 127.1.19.254:45317: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45317: connect: Connection refused (error 111),
16:08:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.196053 1394 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:08:01.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.209285 1418 tablet.cc:2369] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:08:01.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.422909 1466 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:08:01.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.423502 1466 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:08:01.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.424358 1466 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:08:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.458135 1466 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:08:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.458455 1466 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:08:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.458918 1466 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:08:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.459262 1466 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:08:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.459575 1466 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:08:01.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.459918 1466 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:08:01.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.460193 1466 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38015/jwks.json
16:08:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.504452 1466 master_runner.cc:386] Master server non-default flags:
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:43623
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/data
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/wal
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45317,127.1.19.253:45027,127.1.19.252:41153
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kudu_127.1.19.254.keytab
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.254
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.254:45317
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38015/jwks.json
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/data/info.pb
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.254
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42345
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/logs
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.506150 1466 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:08:01.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.507545 1466 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:08:01.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.536633 1490 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:08:01.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.558544 1469 raft_consensus.cc:482] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:08:01.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.559217 1469 raft_consensus.cc:504] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de38a497494a4fe9b7efbda09e530ab3" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45317 } } peers { permanent_uuid: "8d0b5ea683574cf4bf37f74a0010487e" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } } peers { permanent_uuid: "ca4d7764cafd4e12a168e1ccaa7ae84c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } }
16:08:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.565007 1413 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45317: connect: Connection refused (error 111)
16:08:01.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.565995 1469 leader_election.cc:288] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers de38a497494a4fe9b7efbda09e530ab3 (127.1.19.254:45317), 8d0b5ea683574cf4bf37f74a0010487e (127.1.19.253:45027)
16:08:01.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.571031 1413 leader_election.cc:334] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer de38a497494a4fe9b7efbda09e530ab3 (127.1.19.254:45317): Network error: Client connection negotiation failed: client connection to 127.1.19.254:45317: connect: Connection refused (error 111)
16:08:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.573297 1415 leader_election.cc:334] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8d0b5ea683574cf4bf37f74a0010487e (127.1.19.253:45027): Network error: Client connection negotiation failed: client connection to 127.1.19.253:45027: connect: Connection refused (error 111)
16:08:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.574120 1415 leader_election.cc:302] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ca4d7764cafd4e12a168e1ccaa7ae84c; no voters: 8d0b5ea683574cf4bf37f74a0010487e, de38a497494a4fe9b7efbda09e530ab3
16:08:01.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.575596 1469 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:08:02.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:02.182860 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:02.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:02.183351 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:02.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.547509 1492 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:08:02.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:01.536633 1489 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:08:02.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:01.541567 1466 server_base.cc:1034] running on GCE node
16:08:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:02.897610 1466 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:08:02.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:02.901016 1466 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:08:02.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:02.902848 1466 hybrid_clock.cc:648] HybridClock initialized: now 1790438882902757 us; error 112 us; skew 500 ppm
16:08:02.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:02.903815 1466 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:08:02.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:08:02 dist-test-slave-50w2 krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438882, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:08:02.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:02.910044 1466 init.cc:377] Logged in from keytab as kudu/127.1.19.254@KRBTEST.COM (short username kudu)
16:08:02.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:02.915669 1466 webserver.cc:460] Webserver started at http://127.1.19.254:42345/ using document root <none> and password file <none>
16:08:02.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:02.916805 1466 fs_manager.cc:359] Metadata directory not provided
16:08:02.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:02.917070 1466 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:08:02.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:02.926051 1466 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
16:08:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:02.930961 1501 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:08:02.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:02.932379 1466 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
16:08:02.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:02.932787 1466 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/data,/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/wal
16:08:02.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de38a497494a4fe9b7efbda09e530ab3"
16:08:02.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:54 on dist-test-slave-50w2"
16:08:02.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:02.936316 1466 fs_report.cc:383] FS layout report
16:08:02.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:08:02.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/wal
16:08:02.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/wal
16:08:02.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/data/data
16:08:02.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:08:02.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:08:02.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:08:02.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:08:02.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:08:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:08:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:08:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:08:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:08:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:08:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:08:02.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:02.981097 1466 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:08:02.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:02.989025 1466 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:08:02.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:02.989710 1466 kserver.cc:163] Server-wide thread pool size limit: 3276
16:08:03.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:03.079662 1466 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.254:45317
16:08:03.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:03.079840 1552 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.254:45317 every 8 connection(s)
16:08:03.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:03.082943 1466 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/data/info.pb
16:08:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:03.086680 1103 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu as pid 1466
16:08:03.088 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:08:03.088 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.253:45027
16:08:03.088 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
16:08:03.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:03.090468 1103 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:08:03.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu
16:08:03.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/wal
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/data
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/logs
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/data/info.pb
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.253:45027
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.253
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37601
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kudu_127.1.19.253.keytab
16:08:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.253
16:08:03.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:08:03.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:08:03.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:08:03.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:43623
16:08:03.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:08:03.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:08:03.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:08:03.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:08:03.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45317,127.1.19.253:45027,127.1.19.252:41153
16:08:03.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:08:03.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38015/jwks.json
16:08:03.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:08:03.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:03.104135 1553 sys_catalog.cc:263] Verifying existing consensus state
16:08:03.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260926 16:08:03.105229 1553 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
16:08:03.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:03.105736 1466 master.cc:537] Master@127.1.19.254:45317 shutting down...
16:08:03.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:03.189162 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:03.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:03.189545 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:03.194764 1466 master.cc:552] Master@127.1.19.254:45317 shutdown complete.
16:08:03.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
16:08:03.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:03.443048 1555 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:08:03.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:03.443675 1555 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:08:03.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:03.444267 1555 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:08:03.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:03.478412 1555 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:08:03.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:03.478659 1555 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:08:03.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:03.479002 1555 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:08:03.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:03.479211 1555 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:08:03.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:03.479405 1555 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:08:03.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:03.479592 1555 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:08:03.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:03.479722 1555 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38015/jwks.json
16:08:03.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:03.517207 1555 master_runner.cc:386] Master server non-default flags:
16:08:03.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:08:03.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:43623
16:08:03.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:08:03.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:08:03.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/data
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/wal
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45317,127.1.19.253:45027,127.1.19.252:41153
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/krb5kdc/kudu_127.1.19.253.keytab
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.253
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.253:45027
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38015/jwks.json
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/data/info.pb
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.253
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37601
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/logs
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:08:03.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:08:03.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:08:03.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:08:03.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:08:03.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:08:03.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:08:03.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:08:03.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:03.518677 1555 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:08:03.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:03.520068 1555 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:08:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:03.542130 1561 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:08:03.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:03.773020 1567 raft_consensus.cc:482] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:08:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:03.773725 1567 raft_consensus.cc:504] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de38a497494a4fe9b7efbda09e530ab3" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45317 } } peers { permanent_uuid: "8d0b5ea683574cf4bf37f74a0010487e" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } } peers { permanent_uuid: "ca4d7764cafd4e12a168e1ccaa7ae84c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } }
16:08:03.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:03.790503 1567 leader_election.cc:288] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers de38a497494a4fe9b7efbda09e530ab3 (127.1.19.254:45317), 8d0b5ea683574cf4bf37f74a0010487e (127.1.19.253:45027)
16:08:03.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:03.802699 1413 leader_election.cc:334] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer de38a497494a4fe9b7efbda09e530ab3 (127.1.19.254:45317): Network error: Client connection negotiation failed: client connection to 127.1.19.254:45317: connect: Connection refused (error 111)
16:08:03.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:03.806583 1415 leader_election.cc:334] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8d0b5ea683574cf4bf37f74a0010487e (127.1.19.253:45027): Network error: Client connection negotiation failed: client connection to 127.1.19.253:45027: connect: Connection refused (error 111)
16:08:03.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:03.807526 1415 leader_election.cc:302] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ca4d7764cafd4e12a168e1ccaa7ae84c; no voters: 8d0b5ea683574cf4bf37f74a0010487e, de38a497494a4fe9b7efbda09e530ab3
16:08:03.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:03.809706 1567 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:08:04.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:04.195322 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:04.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:04.195829 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:04.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:03.542191 1562 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:08:04.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:03.542321 1564 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:08:04.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:04.921802 1563 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1376 milliseconds
16:08:04.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:04.921927 1555 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:08:04.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:04.923332 1555 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:08:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:04.925850 1555 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:08:04.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:04.927228 1555 hybrid_clock.cc:648] HybridClock initialized: now 1790438884927196 us; error 80 us; skew 500 ppm
16:08:04.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:04.928272 1555 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:08:04.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:08:04 dist-test-slave-50w2 krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438884, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:08:04.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:04.934917 1555 init.cc:377] Logged in from keytab as kudu/127.1.19.253@KRBTEST.COM (short username kudu)
16:08:04.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:04.940449 1555 webserver.cc:460] Webserver started at http://127.1.19.253:37601/ using document root <none> and password file <none>
16:08:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:04.941365 1555 fs_manager.cc:359] Metadata directory not provided
16:08:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:04.941687 1555 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:08:04.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:04.951200 1555 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.008s sys 0.000s
16:08:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:04.956406 1577 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:08:04.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:04.960076 1555 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.005s sys 0.000s
16:08:04.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:04.960728 1555 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/data,/tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/wal
16:08:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8d0b5ea683574cf4bf37f74a0010487e"
16:08:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:07:56 on dist-test-slave-50w2"
16:08:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:04.963316 1555 fs_report.cc:383] FS layout report
16:08:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:08:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/wal
16:08:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/wal
16:08:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/data/data
16:08:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:08:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:08:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:08:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:08:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:08:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:08:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:08:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:08:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:08:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:08:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:08:04.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:04.998203 1555 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:08:05.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:05.004400 1555 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:08:05.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:05.004938 1555 kserver.cc:163] Server-wide thread pool size limit: 3276
16:08:05.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:05.079954 1555 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.253:45027
16:08:05.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:05.080112 1628 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.253:45027 every 8 connection(s)
16:08:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:05.082872 1555 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/data/info.pb
16:08:05.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:05.088701 1103 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu as pid 1555
16:08:05.089 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:08:05.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:05.094790 1629 sys_catalog.cc:263] Verifying existing consensus state
16:08:05.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260926 16:08:05.095829 1629 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
16:08:05.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:05.096293 1555 master.cc:537] Master@127.1.19.253:45027 shutting down...
16:08:05.152 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.252:41153(127.1.19.252:41153)] connecting to peer
16:08:05.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:05.160781 1555 master.cc:552] Master@127.1.19.253:45027 shutdown complete.
16:08:05.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskmIbLz2/test-tmp/mini-kudu-cluster5055649984459981019/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
16:08:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:05.198947 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:05.199293 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:05.222 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:05.223 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:05.281 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.19.252:41153(127.1.19.252:41153)] Successfully connected to peer
16:08:05.284 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.287 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.339 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.355 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
16:08:05.364 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.19.252
16:08:05.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:08:05 dist-test-slave-50w2 krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438879, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.252@KRBTEST.COM
16:08:05.598 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xa55dcc12, L:/127.0.0.1:48250 - R:/127.1.19.252:41153] using SASL/GSSAPI
16:08:05.615 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:05.639 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:05.643 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.643 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.644 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.644 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:05.647 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.647 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.648 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.648 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:05.656 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:05.656 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:05.659 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.659 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.659 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.659 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.660 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.660 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.662 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:05.676 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:05.681 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:05.683 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.684 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.686 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.688 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:05.715 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:05.716 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:05.720 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.720 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.721 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.724 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.724 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.725 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:05.755 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:05.758 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:05.763 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.763 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.768 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.768 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.769 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:05.835 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:05.837 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.838 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:05.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.839 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.841 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.842 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.842 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.842 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:05.935 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:05.937 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:05.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.939 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:05.941 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.941 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.941 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:05.942 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:06.095 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:06.096 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:06.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:06.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:06.099 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:06.103 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:06.104 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:06.104 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:06.105 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:06.195 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:06.196 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:06.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:06.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:06.199 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:06.201 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:06.201 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:06.202 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:06.202 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:06.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:06.204226 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:06.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:06.205089 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:06.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:06.205349 1394 master_runner.cc:186] Error getting master registration for 127.1.19.254:45317: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45317: connect: Connection refused (error 111),
16:08:06.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:06.205837 1394 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:08:06.356 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:06.356 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:06.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:06.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:06.359 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:06.360 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:06.361 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:06.361 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:06.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:06.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:06.747257 1637 raft_consensus.cc:482] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:08:06.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:06.747722 1637 raft_consensus.cc:504] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de38a497494a4fe9b7efbda09e530ab3" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45317 } } peers { permanent_uuid: "8d0b5ea683574cf4bf37f74a0010487e" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } } peers { permanent_uuid: "ca4d7764cafd4e12a168e1ccaa7ae84c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } }
16:08:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:06.749816 1637 leader_election.cc:288] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers de38a497494a4fe9b7efbda09e530ab3 (127.1.19.254:45317), 8d0b5ea683574cf4bf37f74a0010487e (127.1.19.253:45027)
16:08:06.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:06.753278 1413 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45317: connect: Connection refused (error 111) [suppressed 3 similar messages]
16:08:06.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:06.757766 1415 leader_election.cc:334] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8d0b5ea683574cf4bf37f74a0010487e (127.1.19.253:45027): Network error: Client connection negotiation failed: client connection to 127.1.19.253:45027: connect: Connection refused (error 111)
16:08:06.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:06.758342 1413 leader_election.cc:334] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer de38a497494a4fe9b7efbda09e530ab3 (127.1.19.254:45317): Network error: Client connection negotiation failed: client connection to 127.1.19.254:45317: connect: Connection refused (error 111)
16:08:06.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:06.758816 1413 leader_election.cc:302] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ca4d7764cafd4e12a168e1ccaa7ae84c; no voters: 8d0b5ea683574cf4bf37f74a0010487e, de38a497494a4fe9b7efbda09e530ab3
16:08:06.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:06.759716 1637 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:08:07.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:07.208673 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:07.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:07.208953 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:07.696 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:07.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:07.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:07.698 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:07.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:07.701 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:07.701 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:07.701 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:07.702 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:07.855 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:07.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:07.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:07.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:07.859 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:07.861 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:07.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:07.861 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:07.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:08.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:08.211925 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:08.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:08.212183 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:09.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:09.216673 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:09.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:09.217871 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:10.225770 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:10.226110 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:10.228444 944 debug-util.cc:398] Leaking SignalData structure 0x7b08000af060 after lost signal to thread 818
16:08:10.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:10.229337 944 debug-util.cc:398] Leaking SignalData structure 0x7b08000af400 after lost signal to thread 947
16:08:10.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:08:10 dist-test-slave-50w2 krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438890, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:08:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:10.878675 1408 init.cc:295] Successfully reacquired a new kerberos TGT
16:08:11.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:11.175748 1649 raft_consensus.cc:482] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:08:11.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:11.176309 1649 raft_consensus.cc:504] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de38a497494a4fe9b7efbda09e530ab3" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45317 } } peers { permanent_uuid: "8d0b5ea683574cf4bf37f74a0010487e" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } } peers { permanent_uuid: "ca4d7764cafd4e12a168e1ccaa7ae84c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } }
16:08:11.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:11.178551 1649 leader_election.cc:288] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers de38a497494a4fe9b7efbda09e530ab3 (127.1.19.254:45317), 8d0b5ea683574cf4bf37f74a0010487e (127.1.19.253:45027)
16:08:11.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:11.184614 1413 leader_election.cc:334] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer de38a497494a4fe9b7efbda09e530ab3 (127.1.19.254:45317): Network error: Client connection negotiation failed: client connection to 127.1.19.254:45317: connect: Connection refused (error 111)
16:08:11.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:11.186583 1415 leader_election.cc:334] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8d0b5ea683574cf4bf37f74a0010487e (127.1.19.253:45027): Network error: Client connection negotiation failed: client connection to 127.1.19.253:45027: connect: Connection refused (error 111)
16:08:11.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:11.187080 1415 leader_election.cc:302] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ca4d7764cafd4e12a168e1ccaa7ae84c; no voters: 8d0b5ea683574cf4bf37f74a0010487e, de38a497494a4fe9b7efbda09e530ab3
16:08:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:11.188024 1649 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:08:11.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:11.208515 1394 master_runner.cc:186] Error getting master registration for 127.1.19.254:45317: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45317: connect: Connection refused (error 111),
16:08:11.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:11.209044 1394 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:08:11.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:11.229185 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:11.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:11.229477 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:11.755 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:11.756 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:11.757 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:11.757 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:11.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:11.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:11.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:11.759 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:11.760 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:12.232329 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:12.232661 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:12.275 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:12.277 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:12.277 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:12.278 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:12.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:12.280 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:12.280 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:12.280 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:12.280 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:13.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:13.236137 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:13.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:13.236464 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:14.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:14.239642 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:14.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:14.239950 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:14.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:14.245591 525 debug-util.cc:398] Leaking SignalData structure 0x7b08000b21e0 after lost signal to thread 460
16:08:14.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:14.246187 525 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9d60 after lost signal to thread 528
16:08:14.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:14.319826 1657 raft_consensus.cc:482] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:08:14.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:14.320261 1657 raft_consensus.cc:504] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de38a497494a4fe9b7efbda09e530ab3" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45317 } } peers { permanent_uuid: "8d0b5ea683574cf4bf37f74a0010487e" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } } peers { permanent_uuid: "ca4d7764cafd4e12a168e1ccaa7ae84c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } }
16:08:14.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:14.321977 1657 leader_election.cc:288] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers de38a497494a4fe9b7efbda09e530ab3 (127.1.19.254:45317), 8d0b5ea683574cf4bf37f74a0010487e (127.1.19.253:45027)
16:08:14.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:14.324182 1413 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45317: connect: Connection refused (error 111) [suppressed 3 similar messages]
16:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:14.328483 1413 leader_election.cc:334] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer de38a497494a4fe9b7efbda09e530ab3 (127.1.19.254:45317): Network error: Client connection negotiation failed: client connection to 127.1.19.254:45317: connect: Connection refused (error 111)
16:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:14.329027 1415 leader_election.cc:334] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8d0b5ea683574cf4bf37f74a0010487e (127.1.19.253:45027): Network error: Client connection negotiation failed: client connection to 127.1.19.253:45027: connect: Connection refused (error 111)
16:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:14.329488 1415 leader_election.cc:302] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ca4d7764cafd4e12a168e1ccaa7ae84c; no voters: 8d0b5ea683574cf4bf37f74a0010487e, de38a497494a4fe9b7efbda09e530ab3
16:08:14.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:14.330161 1657 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:08:15.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:15.038146 450 debug-util.cc:398] Leaking SignalData structure 0x7b080008f580 after lost signal to thread 387
16:08:15.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:15.038975 450 debug-util.cc:398] Leaking SignalData structure 0x7b08000a88c0 after lost signal to thread 453
16:08:15.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:15.245081 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:15.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:15.245438 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:15.735 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:15.736 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:15.739 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:15.740 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:15.740 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:15.742 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:15.742 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:15.742 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:15.742 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:16.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:16.211858 1394 master_runner.cc:186] Error getting master registration for 127.1.19.254:45317: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45317: connect: Connection refused (error 111),
16:08:16.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:16.212177 1394 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:08:16.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:16.248452 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:16.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:16.248693 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:17.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:17.251529 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:17.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:17.251793 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:17.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:17.561848 1667 raft_consensus.cc:482] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:08:17.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:17.562239 1667 raft_consensus.cc:504] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de38a497494a4fe9b7efbda09e530ab3" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45317 } } peers { permanent_uuid: "8d0b5ea683574cf4bf37f74a0010487e" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 45027 } } peers { permanent_uuid: "ca4d7764cafd4e12a168e1ccaa7ae84c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 41153 } }
16:08:17.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:17.563885 1667 leader_election.cc:288] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers de38a497494a4fe9b7efbda09e530ab3 (127.1.19.254:45317), 8d0b5ea683574cf4bf37f74a0010487e (127.1.19.253:45027)
16:08:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:17.568789 1413 leader_election.cc:334] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer de38a497494a4fe9b7efbda09e530ab3 (127.1.19.254:45317): Network error: Client connection negotiation failed: client connection to 127.1.19.254:45317: connect: Connection refused (error 111)
16:08:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:17.569298 1415 leader_election.cc:334] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8d0b5ea683574cf4bf37f74a0010487e (127.1.19.253:45027): Network error: Client connection negotiation failed: client connection to 127.1.19.253:45027: connect: Connection refused (error 111)
16:08:17.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:17.569743 1415 leader_election.cc:302] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ca4d7764cafd4e12a168e1ccaa7ae84c; no voters: 8d0b5ea683574cf4bf37f74a0010487e, de38a497494a4fe9b7efbda09e530ab3
16:08:17.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:17.570516 1667 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:08:18.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:18.254858 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:18.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:18.255111 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:18.916 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:18.917 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:18.919 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:18.919 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:18.919 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:18.922 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:18.922 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:18.922 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:18.923 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:19.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:19.258455 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:19.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:19.258708 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:20.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:20.262390 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:20.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:20.262723 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:20.375 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:20.376 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:20.379 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:20.380 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:20.380 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:20.380 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:20.380 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:20.380 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:20.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:20.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:08:20 dist-test-slave-50w2 krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438900, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:08:20.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:20.881533 1408 init.cc:295] Successfully reacquired a new kerberos TGT
16:08:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:21.215592 1394 master_runner.cc:186] Error getting master registration for 127.1.19.254:45317: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45317: connect: Connection refused (error 111),
16:08:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:21.216009 1394 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:08:21.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:21.265509 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:21.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:21.265782 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:22.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:22.268639 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:22.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:22.268896 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:23.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:23.272056 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:23.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:23.272337 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:23.896 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:23.897 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:23.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:23.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:23.898 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:23.901 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:23.901 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:23.901 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:23.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:24.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:24.275591 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:24.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:24.275856 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:25.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:25.279147 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:25.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:25.279382 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:25.336 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:25.338 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:25.339 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:25.339 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:25.339 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:25.339 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:25.339 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:25.339 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:25.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:26.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:26.219307 1394 master_runner.cc:186] Error getting master registration for 127.1.19.254:45317: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45317: connect: Connection refused (error 111),
16:08:26.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:26.219755 1394 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:08:26.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:26.282896 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:26.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:26.283177 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:27.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:27.286396 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:27.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:27.286643 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:28.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:28.289536 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:28.289836 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:28.955 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:28.956 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:28.957 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:28.957 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:28.957 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:28.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:28.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:28.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:28.959 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:29.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:29.293012 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:29.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:29.293296 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:30.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:30.296916 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:30.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:30.297235 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:30.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:08:30 dist-test-slave-50w2 krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438910, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:08:30.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:30.883982 1408 init.cc:295] Successfully reacquired a new kerberos TGT
16:08:31.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:31.223444 1394 master_runner.cc:186] Error getting master registration for 127.1.19.254:45317: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45317: connect: Connection refused (error 111),
16:08:31.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:31.223963 1394 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:08:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:31.299950 1480 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca4d7764cafd4e12a168e1ccaa7ae84c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:08:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:08:31.300232 1480 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:08:31.775 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45317(127.1.19.254:45317)] connecting to peer
16:08:31.777 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:45027(127.1.19.253:45027)] connecting to peer
16:08:31.777 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:31.777 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45317(127.1.19.254:45317)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:31.777 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45317: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317
16:08:31.780 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:31.780 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:45027(127.1.19.253:45027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:31.780 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:45027: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
16:08:31.780 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027; will retry
16:08:31.785 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=26687), Traces: [0ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [187ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [239ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [513ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [517ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [536ms] refreshing cache from master, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [540ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [544ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [545ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [548ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [550ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [555ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [560ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [560ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [562ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [563ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [585ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [586ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [588ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [589ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [622ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [624ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [625ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [626ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [665ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [667ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [669ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [670ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [739ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [740ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [743ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [744ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [835ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [839ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [840ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [842ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [843ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [995ms] refreshing cache from master, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [1003ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [1005ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [1007ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [1095ms] refreshing cache from master, [1096ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [1096ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [1097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [1099ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [1101ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [1102ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [1103ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [1256ms] refreshing cache from master, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [1257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [1257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [1261ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [1262ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [1262ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [2596ms] refreshing cache from master, [2596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [2596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [2599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [2599ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [2601ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [2602ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [2603ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [2756ms] refreshing cache from master, [2756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [2756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [2759ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [2760ms] too many traces: truncated at 100 traces, deferred=Deferred@1985091953(state=PENDING, result=null, callback=(continuation of Deferred@314671856 after retry RPC after error@322463650) -> (continuation of Deferred@749821529 after retry RPC after error@1246642991) -> (continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995), errback=(continuation of Deferred@314671856 after retry RPC after error@322463650) -> (continuation of Deferred@749821529 after retry RPC after error@1246642991) -> (continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995)))
16:08:31.791 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.253:45027(127.1.19.253:45027)] disconnecting while in state TERMINATED
16:08:31.792 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.254:45317(127.1.19.254:45317)] disconnecting while in state TERMINATED
16:08:31.793 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.252:41153(127.1.19.252:41153)] disconnecting while in state READY
16:08:31.799 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:08:31.801 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.19.252:41153(127.1.19.252:41153)] handling channelInactive
16:08:31.801 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.252:41153(127.1.19.252:41153)] cleaning up while in state READY due to: connection closed
16:08:31.803 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:08:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:31.808475 1103 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu with pid 1394
16:08:32.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T16:08:32Z chronyd exiting
16:08:34.033 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:08:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:34.035161 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu with pid 654
16:08:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:34.061309 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu with pid 812
16:08:34.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:34.088541 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu with pid 958
16:08:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:34.116204 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu with pid 386
16:08:34.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:34.151717 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu with pid 456
16:08:34.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:08:34.183089 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmIbLz2/build/tsan/bin/kudu with pid 531
16:08:35.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T16:08:35Z chronyd exiting
16:08:36.359 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=26687), Traces: [0ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [187ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [239ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [513ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [517ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [536ms] refreshing cache from master, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [540ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [544ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [545ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [548ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [550ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [555ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [560ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [560ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [562ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [563ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [585ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [586ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [588ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [589ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [622ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [624ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [625ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [626ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [665ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [667ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [669ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [670ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [739ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [740ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [743ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [744ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [835ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [839ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [840ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [842ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [843ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [995ms] refreshing cache from master, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [1003ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [1005ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [1007ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [1095ms] refreshing cache from master, [1096ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [1096ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [1097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [1099ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [1101ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [1102ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [1103ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [1256ms] refreshing cache from master, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [1257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [1257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [1261ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [1262ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [1262ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [2596ms] refreshing cache from master, [2596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [2596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [2599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [2599ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [2601ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [2602ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [2603ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [2756ms] refreshing cache from master, [2756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [2756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [2759ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [2760ms] too many traces: truncated at 100 traces, deferred=Deferred@1985091953(state=PENDING, result=null, callback=(continuation of Deferred@314671856 after retry RPC after error@322463650) -> (continuation of Deferred@749821529 after retry RPC after error@1246642991) -> (continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995), errback=(continuation of Deferred@314671856 after retry RPC after error@322463650) -> (continuation of Deferred@749821529 after retry RPC after error@1246642991) -> (continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:475) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:566) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:475) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:566) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:566) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.253:45027
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=26687), Traces: [0ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [187ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [239ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [513ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [517ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [536ms] refreshing cache from master, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [540ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [544ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [545ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [548ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [550ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [555ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [560ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [560ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [562ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [563ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [585ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [586ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [588ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [589ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [622ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [624ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [625ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [626ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [665ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [667ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [669ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [670ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [739ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [740ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [743ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [744ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [835ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [839ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [840ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [842ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [843ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [995ms] refreshing cache from master, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [1003ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [1005ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [1007ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [1095ms] refreshing cache from master, [1096ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [1096ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [1097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [1099ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [1101ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [1102ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [1103ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [1256ms] refreshing cache from master, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [1257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [1257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [1261ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [1262ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [1262ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [2596ms] refreshing cache from master, [2596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [2596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [2599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:45027, [2599ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [2601ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:41153: OK, [2602ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:45027: Network error: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [2603ms] delaying RPC due to: Service unavailable: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027, [2756ms] refreshing cache from master, [2756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:41153, [2756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45317, [2759ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45317: Network error: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317, [2760ms] too many traces: truncated at 100 traces, deferred=Deferred@1985091953(state=PENDING, result=null, callback=(continuation of Deferred@314671856 after retry RPC after error@322463650) -> (continuation of Deferred@749821529 after retry RPC after error@1246642991) -> (continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995), errback=(continuation of Deferred@314671856 after retry RPC after error@322463650) -> (continuation of Deferred@749821529 after retry RPC after error@1246642991) -> (continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:475)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:566)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.252:41153,127.1.19.254:45317,127.1.19.253:45027) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45317(127.1.19.254:45317): Connection refused: /127.1.19.254:45317,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:475)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:566)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:45027(127.1.19.253:45027): Connection refused: /127.1.19.253:45027
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
... 14 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.253:45027
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)