01:52:34.683 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
01:52:34.683 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskHBGiCK/test-tmp
01:52:34.683 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:52:34.684 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskHBGiCK/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:52:34.685 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
01:52:35.013 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:52:35.014 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:52:35.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:35.017023 339 mini_oidc.cc:152] Starting JWKS server
01:52:35.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:35.020774 339 webserver.cc:460] Webserver started at http://127.0.0.1:44191/ using document root <none> and password file <none>
01:52:35.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:35.021277 339 mini_oidc.cc:189] Starting OIDC Discovery server
01:52:35.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:35.022948 339 webserver.cc:460] Webserver started at http://127.0.0.1:39565/ using document root <none> and password file <none>
01:52:35.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:52:35Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:52:35.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:52:35Z Disabled control of system clock
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:35.073904 339 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-0/wal
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-0/data
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-0/logs
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-0/data/info.pb
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:52:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:52:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:52:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:52:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.84.254:33371
01:52:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.84.254
01:52:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:52:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.84.212:33991
01:52:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:52:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:52:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.84.254:33371,127.0.84.253:33715,127.0.84.252:39381
01:52:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:52:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44191/jwks.json with env {}
01:52:35.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:35.383126 351 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:52:35.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:35.383692 351 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:52:35.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:35.384172 351 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:52:35.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:35.415287 351 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:52:35.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:35.415572 351 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:52:35.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:35.415759 351 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:52:35.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:35.415956 351 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:52:35.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:35.416143 351 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:52:35.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:35.416282 351 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44191/jwks.json
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:35.452023 351 master_runner.cc:386] Master server non-default flags:
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.84.212:33991
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-0/data
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-0/wal
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.84.254:33371,127.0.84.253:33715,127.0.84.252:39381
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.84.254:33371
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44191/jwks.json
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-0/data/info.pb
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.84.254
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:52:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:52:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-0/logs
01:52:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:52:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:52:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:52:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:52:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:52:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:52:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:52:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:52:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:52:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:35.453338 351 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:52:35.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:35.454916 351 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:52:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:35.474936 357 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:36.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:35.474958 358 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:36.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:35.476181 351 server_base.cc:1034] running on GCE node
01:52:36.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:35.474967 360 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:36.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.723611 351 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:52:36.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.726866 351 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:52:36.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.728391 351 hybrid_clock.cc:648] HybridClock initialized: now 1782006756728322 us; error 83 us; skew 500 ppm
01:52:36.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.729346 351 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:52:36.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.736810 351 webserver.cc:460] Webserver started at http://127.0.84.254:33193/ using document root <none> and password file <none>
01:52:36.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.737860 351 fs_manager.cc:359] Metadata directory not provided
01:52:36.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.738062 351 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:52:36.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.738498 351 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:52:36.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.743633 351 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-0/data/instance:
01:52:36.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "87ba964456894f25be681fe285069494"
01:52:36.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:36 on dist-test-slave-s8kj"
01:52:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.744868 351 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-0/wal/instance:
01:52:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "87ba964456894f25be681fe285069494"
01:52:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:36 on dist-test-slave-s8kj"
01:52:36.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.753096 351 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
01:52:36.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.759392 367 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:52:36.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.760901 351 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
01:52:36.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.761333 351 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-0/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-0/wal
01:52:36.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "87ba964456894f25be681fe285069494"
01:52:36.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:36 on dist-test-slave-s8kj"
01:52:36.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.761667 351 fs_report.cc:383] FS layout report
01:52:36.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:52:36.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-0/wal
01:52:36.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-0/wal
01:52:36.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-0/data/data
01:52:36.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:52:36.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:52:36.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:52:36.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:52:36.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:52:36.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:52:36.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:52:36.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:52:36.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:52:36.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:52:36.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:52:36.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.798808 351 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:52:36.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.800269 351 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:52:36.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.800673 351 kserver.cc:163] Server-wide thread pool size limit: 3276
01:52:36.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.873998 351 rpc_server.cc:307] RPC server started. Bound to: 127.0.84.254:33371
01:52:36.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.874096 418 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.84.254:33371 every 8 connection(s)
01:52:36.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.876744 351 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-0/data/info.pb
01:52:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.881997 419 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:52:36.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.883271 339 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 351
01:52:36.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.883703 339 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-0/wal/instance
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.887018 339 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-1/wal
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-1/data
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-1/logs
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-1/data/info.pb
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:52:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:52:36.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:52:36.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.84.253:33715
01:52:36.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.84.253
01:52:36.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:52:36.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.84.212:33991
01:52:36.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:36.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:36.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:52:36.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:52:36.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.84.254:33371,127.0.84.253:33715,127.0.84.252:39381
01:52:36.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:52:36.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44191/jwks.json with env {}
01:52:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.895807 419 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } has no permanent_uuid. Determining permanent_uuid...
01:52:36.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.925240 419 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } has no permanent_uuid. Determining permanent_uuid...
01:52:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:36.927641 371 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.84.253:33715: connect: Connection refused (error 111)
01:52:36.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:36.931501 419 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.253:33715: Network error: Client connection negotiation failed: client connection to 127.0.84.253:33715: connect: Connection refused (error 111)
01:52:36.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:36.981154 419 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } attempt: 1
01:52:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:36.985338 419 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.253:33715: Network error: Client connection negotiation failed: client connection to 127.0.84.253:33715: connect: Connection refused (error 111)
01:52:37.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:37.053750 419 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } attempt: 2
01:52:37.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:37.057896 419 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.253:33715: Network error: Client connection negotiation failed: client connection to 127.0.84.253:33715: connect: Connection refused (error 111)
01:52:37.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:37.149317 419 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } attempt: 3
01:52:37.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:37.153461 419 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.253:33715: Network error: Client connection negotiation failed: client connection to 127.0.84.253:33715: connect: Connection refused (error 111)
01:52:37.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:37.206830 421 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:52:37.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:37.207391 421 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:52:37.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:37.207871 421 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:52:37.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:37.240867 421 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:52:37.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:37.241282 421 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:52:37.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:37.241660 421 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:52:37.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:37.241917 421 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:52:37.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:37.242110 421 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:52:37.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:37.242251 421 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44191/jwks.json
01:52:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:37.284241 421 master_runner.cc:386] Master server non-default flags:
01:52:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.84.212:33991
01:52:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:52:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-1/data
01:52:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-1/wal
01:52:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:52:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.84.254:33371,127.0.84.253:33715,127.0.84.252:39381
01:52:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:52:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.84.253:33715
01:52:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:52:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:52:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44191/jwks.json
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-1/data/info.pb
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.84.253
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-1/logs
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:52:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:37.285540 421 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:52:37.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:37.287132 421 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:52:37.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:37.296892 419 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } attempt: 4
01:52:37.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:37.302349 419 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.253:33715: Network error: Client connection negotiation failed: client connection to 127.0.84.253:33715: connect: Connection refused (error 111)
01:52:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:37.307869 431 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:37.601847 419 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } attempt: 5
01:52:37.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:37.606333 419 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.253:33715: Network error: Client connection negotiation failed: client connection to 127.0.84.253:33715: connect: Connection refused (error 111)
01:52:38.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.153779 419 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } attempt: 6
01:52:38.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:38.161921 419 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.253:33715: Network error: Client connection negotiation failed: client connection to 127.0.84.253:33715: connect: Connection refused (error 111)
01:52:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:37.307951 430 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:37.307987 433 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:37.309775 421 server_base.cc:1034] running on GCE node
01:52:38.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.527873 421 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:52:38.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.531158 421 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:52:38.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.532658 421 hybrid_clock.cc:648] HybridClock initialized: now 1782006758532593 us; error 81 us; skew 500 ppm
01:52:38.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.533643 421 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:52:38.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.541309 421 webserver.cc:460] Webserver started at http://127.0.84.253:34465/ using document root <none> and password file <none>
01:52:38.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.542282 421 fs_manager.cc:359] Metadata directory not provided
01:52:38.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.542470 421 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:52:38.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.542879 421 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:52:38.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.547861 421 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-1/data/instance:
01:52:38.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cd99872eead24416b6d993e69b01a4a4"
01:52:38.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:38 on dist-test-slave-s8kj"
01:52:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.549001 421 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-1/wal/instance:
01:52:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cd99872eead24416b6d993e69b01a4a4"
01:52:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:38 on dist-test-slave-s8kj"
01:52:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.556476 421 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
01:52:38.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.562959 442 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:52:38.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.564412 421 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.004s
01:52:38.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.564751 421 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-1/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-1/wal
01:52:38.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cd99872eead24416b6d993e69b01a4a4"
01:52:38.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:38 on dist-test-slave-s8kj"
01:52:38.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.565169 421 fs_report.cc:383] FS layout report
01:52:38.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:52:38.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-1/wal
01:52:38.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-1/wal
01:52:38.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-1/data/data
01:52:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:52:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:52:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:52:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:52:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:52:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:52:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:52:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:52:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:52:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:52:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:52:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.605530 421 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:52:38.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.607030 421 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:52:38.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.607447 421 kserver.cc:163] Server-wide thread pool size limit: 3276
01:52:38.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.681041 421 rpc_server.cc:307] RPC server started. Bound to: 127.0.84.253:33715
01:52:38.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.681154 493 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.84.253:33715 every 8 connection(s)
01:52:38.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.684082 421 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-1/data/info.pb
01:52:38.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.690011 494 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:52:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.694475 339 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 421
01:52:38.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.694965 339 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-1/wal/instance
01:52:38.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.697173 339 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:38.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:38.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-2/wal
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-2/data
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-2/logs
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-2/data/info.pb
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.84.252:39381
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.84.252
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:52:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.84.212:33991
01:52:38.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:38.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:38.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:52:38.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:52:38.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.84.254:33371,127.0.84.253:33715,127.0.84.252:39381
01:52:38.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:52:38.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44191/jwks.json with env {}
01:52:38.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.703475 494 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } has no permanent_uuid. Determining permanent_uuid...
01:52:38.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.732013 494 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } has no permanent_uuid. Determining permanent_uuid...
01:52:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.749426 494 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } has no permanent_uuid. Determining permanent_uuid...
01:52:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:38.752290 443 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.84.252:39381: connect: Connection refused (error 111)
01:52:38.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:38.756681 494 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.252:39381: Network error: Client connection negotiation failed: client connection to 127.0.84.252:39381: connect: Connection refused (error 111)
01:52:38.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.806353 494 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } attempt: 1
01:52:38.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:38.810912 494 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.252:39381: Network error: Client connection negotiation failed: client connection to 127.0.84.252:39381: connect: Connection refused (error 111)
01:52:38.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.879446 494 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } attempt: 2
01:52:38.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:38.884482 494 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.252:39381: Network error: Client connection negotiation failed: client connection to 127.0.84.252:39381: connect: Connection refused (error 111)
01:52:38.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:38.975958 494 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } attempt: 3
01:52:38.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:38.979730 494 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.252:39381: Network error: Client connection negotiation failed: client connection to 127.0.84.252:39381: connect: Connection refused (error 111)
01:52:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.021174 496 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:52:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.021754 496 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:52:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.022228 496 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:52:39.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.054623 496 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:52:39.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.054948 496 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:52:39.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.055143 496 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:52:39.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.055336 496 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:52:39.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.055521 496 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:52:39.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.055661 496 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44191/jwks.json
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:39.092821 496 master_runner.cc:386] Master server non-default flags:
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.84.212:33991
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-2/data
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-2/wal
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.84.254:33371,127.0.84.253:33715,127.0.84.252:39381
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.84.252:39381
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44191/jwks.json
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-2/data/info.pb
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.84.252
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-2/logs
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:52:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:52:39.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:52:39.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:52:39.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:52:39.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:39.094202 496 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:52:39.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:39.095734 496 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:52:39.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.112826 507 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:39.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:39.123191 494 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } attempt: 4
01:52:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.127013 494 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.252:39381: Network error: Client connection negotiation failed: client connection to 127.0.84.252:39381: connect: Connection refused (error 111)
01:52:39.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:39.222373 419 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } attempt: 7
01:52:39.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:39.241164 419 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } has no permanent_uuid. Determining permanent_uuid...
01:52:39.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.247262 419 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.252:39381: Network error: Client connection negotiation failed: client connection to 127.0.84.252:39381: connect: Connection refused (error 111)
01:52:39.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:39.305967 419 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } attempt: 1
01:52:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.310114 419 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.252:39381: Network error: Client connection negotiation failed: client connection to 127.0.84.252:39381: connect: Connection refused (error 111)
01:52:39.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:39.391793 419 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } attempt: 2
01:52:39.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.396453 419 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.252:39381: Network error: Client connection negotiation failed: client connection to 127.0.84.252:39381: connect: Connection refused (error 111)
01:52:39.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:39.426528 494 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } attempt: 5
01:52:39.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.430483 494 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.252:39381: Network error: Client connection negotiation failed: client connection to 127.0.84.252:39381: connect: Connection refused (error 111)
01:52:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:39.482100 419 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } attempt: 3
01:52:39.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.487059 419 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.252:39381: Network error: Client connection negotiation failed: client connection to 127.0.84.252:39381: connect: Connection refused (error 111)
01:52:39.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:39.627594 419 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } attempt: 4
01:52:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.632155 419 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.252:39381: Network error: Client connection negotiation failed: client connection to 127.0.84.252:39381: connect: Connection refused (error 111)
01:52:39.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:39.915601 419 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } attempt: 5
01:52:39.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.920503 419 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.252:39381: Network error: Client connection negotiation failed: client connection to 127.0.84.252:39381: connect: Connection refused (error 111)
01:52:39.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:39.977965 494 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } attempt: 6
01:52:39.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.985098 494 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.252:39381: Network error: Client connection negotiation failed: client connection to 127.0.84.252:39381: connect: Connection refused (error 111)
01:52:40.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.112815 509 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:40.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:39.113442 506 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:40.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:40.378058 508 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:52:40.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.378180 496 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:52:40.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.383230 496 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:52:40.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.386134 496 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:52:40.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.387571 496 hybrid_clock.cc:648] HybridClock initialized: now 1782006760387537 us; error 69 us; skew 500 ppm
01:52:40.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.388437 496 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:52:40.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.395390 496 webserver.cc:460] Webserver started at http://127.0.84.252:37387/ using document root <none> and password file <none>
01:52:40.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.396334 496 fs_manager.cc:359] Metadata directory not provided
01:52:40.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.396538 496 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:52:40.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.396992 496 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:52:40.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.402259 496 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-2/data/instance:
01:52:40.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "20339e4a31c44b6893956829fe06b3e5"
01:52:40.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:40 on dist-test-slave-s8kj"
01:52:40.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.403437 496 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-2/wal/instance:
01:52:40.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "20339e4a31c44b6893956829fe06b3e5"
01:52:40.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:40 on dist-test-slave-s8kj"
01:52:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.411386 496 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
01:52:40.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.417455 520 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:52:40.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.418843 496 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
01:52:40.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.419242 496 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-2/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-2/wal
01:52:40.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "20339e4a31c44b6893956829fe06b3e5"
01:52:40.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:40 on dist-test-slave-s8kj"
01:52:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.419739 496 fs_report.cc:383] FS layout report
01:52:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:52:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-2/wal
01:52:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-2/wal
01:52:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-2/data/data
01:52:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:52:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:52:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:52:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:52:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:52:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:52:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:52:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:52:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:52:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:52:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:52:40.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.472982 419 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } attempt: 6
01:52:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:40.479244 419 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.84.252:39381: Network error: Client connection negotiation failed: client connection to 127.0.84.252:39381: connect: Connection refused (error 111)
01:52:40.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.483692 496 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:52:40.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.485586 496 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:52:40.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.486078 496 kserver.cc:163] Server-wide thread pool size limit: 3276
01:52:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.560719 496 rpc_server.cc:307] RPC server started. Bound to: 127.0.84.252:39381
01:52:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.560912 573 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.84.252:39381 every 8 connection(s)
01:52:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.563756 496 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-2/data/info.pb
01:52:40.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.569244 574 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:52:40.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.571099 339 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 496
01:52:40.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.571527 339 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/master-2/wal/instance
01:52:40.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.583251 574 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } has no permanent_uuid. Determining permanent_uuid...
01:52:40.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.618921 574 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } has no permanent_uuid. Determining permanent_uuid...
01:52:40.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.632054 574 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } has no permanent_uuid. Determining permanent_uuid...
01:52:40.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.649888 574 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5: Bootstrap starting.
01:52:40.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.656616 574 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5: Neither blocks nor log segments found. Creating new log.
01:52:40.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.658448 574 log.cc:824] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5: Log is configured to *not* fsync() on all Append() calls
01:52:40.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.664671 574 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5: No bootstrap required, opened a new log
01:52:40.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.687033 574 raft_consensus.cc:348] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } }
01:52:40.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.687644 574 raft_consensus.cc:374] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:52:40.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.687839 574 raft_consensus.cc:729] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 20339e4a31c44b6893956829fe06b3e5, State: Initialized, Role: FOLLOWER
01:52:40.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.688570 574 consensus_queue.cc:260] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } }
01:52:40.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.691674 583 sys_catalog.cc:455] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } } }
01:52:40.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.692754 583 sys_catalog.cc:458] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [sys.catalog]: This master's current role is: FOLLOWER
01:52:40.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.693631 574 sys_catalog.cc:564] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [sys.catalog]: configured and running, proceeding with master startup.
01:52:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:40.723311 594 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:52:40.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:40.723745 594 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:52:40.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:40.741496 528 tablet.cc:2369] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:52:40.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.719504 496 master_runner.cc:186] Error getting master registration for 127.0.84.254:33371: OK, instance_id { permanent_uuid: "87ba964456894f25be681fe285069494" instance_seqno: 1782006756854687 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:52:40.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.745116 496 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:52:40.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.939577 583 raft_consensus.cc:482] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:52:40.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.940214 583 raft_consensus.cc:504] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } }
01:52:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:40.942237 583 leader_election.cc:288] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 87ba964456894f25be681fe285069494 (127.0.84.254:33371), cd99872eead24416b6d993e69b01a4a4 (127.0.84.253:33715)
01:52:41.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.045814 494 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } attempt: 7
01:52:41.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.071419 494 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4: Bootstrap starting.
01:52:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.078578 494 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4: Neither blocks nor log segments found. Creating new log.
01:52:41.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.080974 494 log.cc:824] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4: Log is configured to *not* fsync() on all Append() calls
01:52:41.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.087970 494 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4: No bootstrap required, opened a new log
01:52:41.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.119459 494 raft_consensus.cc:348] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } }
01:52:41.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.120342 494 raft_consensus.cc:374] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:52:41.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.120662 494 raft_consensus.cc:729] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd99872eead24416b6d993e69b01a4a4, State: Initialized, Role: FOLLOWER
01:52:41.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.121640 494 consensus_queue.cc:260] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } }
01:52:41.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.124567 598 sys_catalog.cc:455] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } } }
01:52:41.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.125546 598 sys_catalog.cc:458] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4 [sys.catalog]: This master's current role is: FOLLOWER
01:52:41.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.126574 494 sys_catalog.cc:564] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4 [sys.catalog]: configured and running, proceeding with master startup.
01:52:41.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.126782 469 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "20339e4a31c44b6893956829fe06b3e5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd99872eead24416b6d993e69b01a4a4" is_pre_election: true
01:52:41.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.127699 469 raft_consensus.cc:2454] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 20339e4a31c44b6893956829fe06b3e5 in term 0.
01:52:41.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.129165 524 leader_election.cc:302] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 20339e4a31c44b6893956829fe06b3e5, cd99872eead24416b6d993e69b01a4a4; no voters:
01:52:41.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.130082 583 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [term 0 FOLLOWER]: Leader pre-election won for term 1
01:52:41.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.130436 583 raft_consensus.cc:482] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
01:52:41.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.130765 583 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [term 0 FOLLOWER]: Advancing to term 1
01:52:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.136142 583 raft_consensus.cc:504] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } }
01:52:41.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.138038 583 leader_election.cc:288] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [CANDIDATE]: Term 1 election: Requested vote from peers 87ba964456894f25be681fe285069494 (127.0.84.254:33371), cd99872eead24416b6d993e69b01a4a4 (127.0.84.253:33715)
01:52:41.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.140287 469 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "20339e4a31c44b6893956829fe06b3e5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd99872eead24416b6d993e69b01a4a4"
01:52:41.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.140900 469 raft_consensus.cc:3037] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4 [term 0 FOLLOWER]: Advancing to term 1
01:52:41.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.147598 469 raft_consensus.cc:2454] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 20339e4a31c44b6893956829fe06b3e5 in term 1.
01:52:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.149006 524 leader_election.cc:302] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 20339e4a31c44b6893956829fe06b3e5, cd99872eead24416b6d993e69b01a4a4; no voters:
01:52:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.149895 583 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [term 1 FOLLOWER]: Leader election won for term 1
01:52:41.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.157984 583 raft_consensus.cc:686] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [term 1 LEADER]: Becoming Leader. State: Replica: 20339e4a31c44b6893956829fe06b3e5, State: Running, Role: LEADER
01:52:41.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.159930 583 consensus_queue.cc:237] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } }
01:52:41.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.171696 604 sys_catalog.cc:455] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 20339e4a31c44b6893956829fe06b3e5. Latest consensus state: current_term: 1 leader_uuid: "20339e4a31c44b6893956829fe06b3e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } } }
01:52:41.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.172621 604 sys_catalog.cc:458] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [sys.catalog]: This master's current role is: LEADER
01:52:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.183146 612 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
01:52:41.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.190691 612 catalog_manager.cc:1484] Initializing Kudu cluster ID...
01:52:41.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:41.206331 611 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:52:41.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:41.206918 611 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:52:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.224745 469 raft_consensus.cc:1261] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4 [term 1 FOLLOWER]: Refusing update from remote peer 20339e4a31c44b6893956829fe06b3e5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:52:41.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.227109 604 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:52:41.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.201390 421 master_runner.cc:186] Error getting master registration for 127.0.84.254:33371: OK, instance_id { permanent_uuid: "87ba964456894f25be681fe285069494" instance_seqno: 1782006756854687 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:52:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.239804 421 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:52:41.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.283272 598 sys_catalog.cc:455] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 20339e4a31c44b6893956829fe06b3e5. Latest consensus state: current_term: 1 leader_uuid: "20339e4a31c44b6893956829fe06b3e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } } }
01:52:41.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.284202 598 sys_catalog.cc:458] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4 [sys.catalog]: This master's current role is: FOLLOWER
01:52:41.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.288719 614 mvcc.cc:205] Tried to move back new op lower bound from 7299099693955108864 to 7299099693722664960. Current Snapshot: MvccSnapshot[applied={T|T < 7299099693955108864}]
01:52:41.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.291850 604 sys_catalog.cc:455] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "20339e4a31c44b6893956829fe06b3e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } } }
01:52:41.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.292755 604 sys_catalog.cc:458] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [sys.catalog]: This master's current role is: LEADER
01:52:41.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.298327 612 catalog_manager.cc:1347] Generated new cluster ID: 1930ce5279f34f8a852f632e8373a057
01:52:41.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.298600 612 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
01:52:41.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.307188 598 sys_catalog.cc:455] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "20339e4a31c44b6893956829fe06b3e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } } }
01:52:41.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.308012 598 sys_catalog.cc:458] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4 [sys.catalog]: This master's current role is: FOLLOWER
01:52:41.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.359625 612 catalog_manager.cc:1370] Generated new certificate authority record
01:52:41.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.370102 612 catalog_manager.cc:1504] Loading token signing keys...
01:52:41.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.405357 612 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5: Generated new TSK 0
01:52:41.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.406656 612 catalog_manager.cc:1514] Initializing in-progress tserver states...
01:52:41.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.459281 339 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:41.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:41.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-0/wal
01:52:41.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-0/data
01:52:41.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:52:41.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:52:41.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-0/logs
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-0/data/info.pb
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.84.193:0
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.84.193
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.84.193
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.84.254:33371,127.0.84.253:33715,127.0.84.252:39381
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.84.212:33991
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:52:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.512625 419 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } attempt: 7
01:52:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.568645 419 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494: Bootstrap starting.
01:52:41.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.588655 419 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494: Neither blocks nor log segments found. Creating new log.
01:52:41.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.595358 419 log.cc:824] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494: Log is configured to *not* fsync() on all Append() calls
01:52:41.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.609751 419 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494: No bootstrap required, opened a new log
01:52:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.696513 419 raft_consensus.cc:348] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } }
01:52:41.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.699769 419 raft_consensus.cc:374] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:52:41.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.700769 419 raft_consensus.cc:729] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 87ba964456894f25be681fe285069494, State: Initialized, Role: FOLLOWER
01:52:41.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.703307 419 consensus_queue.cc:260] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } }
01:52:41.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.725976 393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "20339e4a31c44b6893956829fe06b3e5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "87ba964456894f25be681fe285069494"
01:52:41.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.726727 393 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494 [term 0 FOLLOWER]: Advancing to term 1
01:52:41.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.729552 394 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "20339e4a31c44b6893956829fe06b3e5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "87ba964456894f25be681fe285069494" is_pre_election: true
01:52:41.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.736411 393 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 20339e4a31c44b6893956829fe06b3e5 in term 1.
01:52:41.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.737720 392 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494 [term 1 FOLLOWER]: Refusing update from remote peer 20339e4a31c44b6893956829fe06b3e5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:52:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.744451 604 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:52:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.746768 419 sys_catalog.cc:564] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494 [sys.catalog]: configured and running, proceeding with master startup.
01:52:41.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.743804 622 sys_catalog.cc:455] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } } }
01:52:41.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.748948 622 sys_catalog.cc:458] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494 [sys.catalog]: This master's current role is: FOLLOWER
01:52:41.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.823627 622 sys_catalog.cc:455] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 20339e4a31c44b6893956829fe06b3e5. Latest consensus state: current_term: 1 leader_uuid: "20339e4a31c44b6893956829fe06b3e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } } }
01:52:41.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.823791 623 sys_catalog.cc:455] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "20339e4a31c44b6893956829fe06b3e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } } }
01:52:41.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.824435 622 sys_catalog.cc:458] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494 [sys.catalog]: This master's current role is: FOLLOWER
01:52:41.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.824483 623 sys_catalog.cc:458] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494 [sys.catalog]: This master's current role is: FOLLOWER
01:52:41.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:41.842022 638 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
01:52:42.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:41.999780 619 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:52:42.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:42.000268 619 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:52:42.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:42.000749 619 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:52:42.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:42.033533 619 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:52:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:42.034416 619 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.84.193
01:52:42.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:42.071782 619 tablet_server_runner.cc:78] Tablet server non-default flags:
01:52:42.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:42.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.84.212:33991
01:52:42.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:42.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:52:42.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-0/data
01:52:42.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-0/wal
01:52:42.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:42.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:42.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.84.193:0
01:52:42.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:52:42.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:42.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:42.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-0/data/info.pb
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.84.193
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.84.254:33371,127.0.84.253:33715,127.0.84.252:39381
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.84.193
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-0/logs
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:52:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:42.073024 619 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:52:42.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:42.075096 619 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:52:42.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:42.100402 644 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:42.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:42.212965 611 catalog_manager.cc:1259] Loaded cluster ID: 1930ce5279f34f8a852f632e8373a057
01:52:42.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:42.213507 611 catalog_manager.cc:1552] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4: loading cluster ID for follower catalog manager: success
01:52:42.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:42.221390 611 catalog_manager.cc:1574] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4: acquiring CA information for follower catalog manager: success
01:52:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:42.229425 611 catalog_manager.cc:1602] T 00000000000000000000000000000000 P cd99872eead24416b6d993e69b01a4a4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:52:42.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:42.266053 604 sys_catalog.cc:455] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "20339e4a31c44b6893956829fe06b3e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "87ba964456894f25be681fe285069494" member_type: VOTER last_known_addr { host: "127.0.84.254" port: 33371 } } peers { permanent_uuid: "cd99872eead24416b6d993e69b01a4a4" member_type: VOTER last_known_addr { host: "127.0.84.253" port: 33715 } } peers { permanent_uuid: "20339e4a31c44b6893956829fe06b3e5" member_type: VOTER last_known_addr { host: "127.0.84.252" port: 39381 } } }
01:52:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:42.267570 604 sys_catalog.cc:458] T 00000000000000000000000000000000 P 20339e4a31c44b6893956829fe06b3e5 [sys.catalog]: This master's current role is: LEADER
01:52:42.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:42.868832 638 catalog_manager.cc:1259] Loaded cluster ID: 1930ce5279f34f8a852f632e8373a057
01:52:42.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:42.869182 638 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494: loading cluster ID for follower catalog manager: success
01:52:42.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:42.903507 638 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494: acquiring CA information for follower catalog manager: success
01:52:42.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:42.909484 638 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 87ba964456894f25be681fe285069494: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:52:44.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:43.494015 643 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 619
01:52:44.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:44.970984 643 kernel_stack_watchdog.cc:198] Thread 619 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 391ms:
01:52:44.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
01:52:44.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
01:52:44.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
01:52:44.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
01:52:44.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:42.101630 645 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:44.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:44.971730 619 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.872s user 0.719s sys 1.143s
01:52:44.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:44.973627 619 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.874s user 0.720s sys 1.143s
01:52:44.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:44.974615 647 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:44.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:44.978323 646 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2878 milliseconds
01:52:44.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:44.979135 619 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:52:44.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:44.980994 619 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:52:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:44.983894 619 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:52:44.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:44.985493 619 hybrid_clock.cc:648] HybridClock initialized: now 1782006764985427 us; error 91 us; skew 500 ppm
01:52:44.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:44.986591 619 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:52:44.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:44.994551 619 webserver.cc:460] Webserver started at http://127.0.84.193:33385/ using document root <none> and password file <none>
01:52:44.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:44.995664 619 fs_manager.cc:359] Metadata directory not provided
01:52:44.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:44.995927 619 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:52:44.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:44.996408 619 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:52:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.001947 619 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-0/data/instance:
01:52:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "167cdb40ec334aa4bcb3a80333f4e111"
01:52:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:44 on dist-test-slave-s8kj"
01:52:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.003260 619 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-0/wal/instance:
01:52:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "167cdb40ec334aa4bcb3a80333f4e111"
01:52:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:44 on dist-test-slave-s8kj"
01:52:45.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.013021 619 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.000s sys 0.007s
01:52:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.019564 659 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:52:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.020845 619 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
01:52:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.021227 619 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-0/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-0/wal
01:52:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "167cdb40ec334aa4bcb3a80333f4e111"
01:52:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:44 on dist-test-slave-s8kj"
01:52:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.021664 619 fs_report.cc:383] FS layout report
01:52:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:52:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-0/wal
01:52:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-0/wal
01:52:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-0/data/data
01:52:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:52:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:52:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:52:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:52:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:52:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:52:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:52:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:52:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:52:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:52:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:52:45.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.071442 619 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:52:45.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.073168 619 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:52:45.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.073710 619 kserver.cc:163] Server-wide thread pool size limit: 3276
01:52:45.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.077153 619 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:52:45.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.082523 619 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:52:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.082849 619 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:52:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.083171 619 ts_tablet_manager.cc:610] Registered 0 tablets
01:52:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.083418 619 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:52:45.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.294960 619 rpc_server.cc:307] RPC server started. Bound to: 127.0.84.193:36895
01:52:45.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.295125 771 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.84.193:36895 every 8 connection(s)
01:52:45.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.297667 619 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-0/data/info.pb
01:52:45.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.308300 339 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 619
01:52:45.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.308954 339 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-0/wal/instance
01:52:45.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.328500 339 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-1/wal
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-1/data
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-1/logs
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-1/data/info.pb
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.84.194:0
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.84.194
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.84.194
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.84.254:33371,127.0.84.253:33715,127.0.84.252:39381
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.84.212:33991
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:52:45.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.346266 772 heartbeater.cc:347] Connected to a master server at 127.0.84.252:39381
01:52:45.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.346958 772 heartbeater.cc:464] Registering TS with master...
01:52:45.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.348498 772 heartbeater.cc:511] Master 127.0.84.252:39381 requested a full tablet report, sending...
01:52:45.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.352571 539 ts_manager.cc:194] Registered new tserver with Master: 167cdb40ec334aa4bcb3a80333f4e111 (127.0.84.193:36895)
01:52:45.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.355571 539 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.84.193:55561
01:52:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.368193 773 heartbeater.cc:347] Connected to a master server at 127.0.84.253:33715
01:52:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.368588 773 heartbeater.cc:464] Registering TS with master...
01:52:45.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.369378 773 heartbeater.cc:511] Master 127.0.84.253:33715 requested a full tablet report, sending...
01:52:45.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.372483 459 ts_manager.cc:194] Registered new tserver with Master: 167cdb40ec334aa4bcb3a80333f4e111 (127.0.84.193:36895)
01:52:45.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.374271 774 heartbeater.cc:347] Connected to a master server at 127.0.84.254:33371
01:52:45.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.374661 774 heartbeater.cc:464] Registering TS with master...
01:52:45.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.375380 774 heartbeater.cc:511] Master 127.0.84.254:33371 requested a full tablet report, sending...
01:52:45.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.378675 384 ts_manager.cc:194] Registered new tserver with Master: 167cdb40ec334aa4bcb3a80333f4e111 (127.0.84.193:36895)
01:52:45.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:45.674521 780 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:52:45.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:45.675000 780 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:52:45.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:45.675437 780 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:52:45.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:45.725314 780 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:52:45.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:45.726624 780 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.84.194
01:52:45.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.771826 780 tablet_server_runner.cc:78] Tablet server non-default flags:
01:52:45.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:45.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.84.212:33991
01:52:45.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:45.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:52:45.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-1/data
01:52:45.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-1/wal
01:52:45.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:45.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:45.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.84.194:0
01:52:45.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:52:45.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:45.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:45.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-1/data/info.pb
01:52:45.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.84.194
01:52:45.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:52:45.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.84.254:33371,127.0.84.253:33715,127.0.84.252:39381
01:52:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:52:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:52:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:52:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.84.194
01:52:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-1/logs
01:52:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:52:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:52:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:52:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:52:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:52:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:52:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:52:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:52:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:52:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.773120 780 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:52:45.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:45.774781 780 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:52:45.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:45.797402 790 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:46.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:46.360625 772 heartbeater.cc:503] Master 127.0.84.252:39381 was elected leader, sending a full tablet report...
01:52:47.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:45.797475 789 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:47.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:45.798942 792 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:47.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:47.134542 791 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1334 milliseconds
01:52:47.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.134704 780 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:52:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.136010 780 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:52:47.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.138358 780 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:52:47.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.139988 780 hybrid_clock.cc:648] HybridClock initialized: now 1782006767139954 us; error 44 us; skew 500 ppm
01:52:47.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.141774 780 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:52:47.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.152449 780 webserver.cc:460] Webserver started at http://127.0.84.194:32877/ using document root <none> and password file <none>
01:52:47.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.153462 780 fs_manager.cc:359] Metadata directory not provided
01:52:47.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.153673 780 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:52:47.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.154065 780 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:52:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.158990 780 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-1/data/instance:
01:52:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5488c47ef19a4a1583a18ae4ab2c1751"
01:52:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:47 on dist-test-slave-s8kj"
01:52:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.160074 780 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-1/wal/instance:
01:52:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5488c47ef19a4a1583a18ae4ab2c1751"
01:52:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:47 on dist-test-slave-s8kj"
01:52:47.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.167397 780 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
01:52:47.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.173388 799 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:52:47.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.174517 780 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
01:52:47.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.174901 780 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-1/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-1/wal
01:52:47.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5488c47ef19a4a1583a18ae4ab2c1751"
01:52:47.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:47 on dist-test-slave-s8kj"
01:52:47.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.175258 780 fs_report.cc:383] FS layout report
01:52:47.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:52:47.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-1/wal
01:52:47.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-1/wal
01:52:47.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-1/data/data
01:52:47.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:52:47.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:52:47.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:52:47.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:52:47.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:52:47.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:52:47.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:52:47.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:52:47.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:52:47.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:52:47.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:52:47.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.221510 780 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:52:47.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.223084 780 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:52:47.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.223632 780 kserver.cc:163] Server-wide thread pool size limit: 3276
01:52:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.226346 780 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:52:47.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.230821 780 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:52:47.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.231065 780 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:52:47.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.231405 780 ts_tablet_manager.cc:610] Registered 0 tablets
01:52:47.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.231587 780 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:52:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.379494 780 rpc_server.cc:307] RPC server started. Bound to: 127.0.84.194:40193
01:52:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.379611 912 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.84.194:40193 every 8 connection(s)
01:52:47.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.382232 780 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-1/data/info.pb
01:52:47.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.384934 339 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 780
01:52:47.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.385462 339 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-1/wal/instance
01:52:47.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.405143 339 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:47.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:47.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-2/wal
01:52:47.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-2/data
01:52:47.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:52:47.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:52:47.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:52:47.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:47.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:47.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:47.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-2/logs
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-2/data/info.pb
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.84.195:0
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.84.195
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.84.195
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.84.254:33371,127.0.84.253:33715,127.0.84.252:39381
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.84.212:33991
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:47.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:52:47.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.437187 913 heartbeater.cc:347] Connected to a master server at 127.0.84.252:39381
01:52:47.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.437716 913 heartbeater.cc:464] Registering TS with master...
01:52:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.439124 913 heartbeater.cc:511] Master 127.0.84.252:39381 requested a full tablet report, sending...
01:52:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.439267 914 heartbeater.cc:347] Connected to a master server at 127.0.84.253:33715
01:52:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.439702 914 heartbeater.cc:464] Registering TS with master...
01:52:47.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.440828 914 heartbeater.cc:511] Master 127.0.84.253:33715 requested a full tablet report, sending...
01:52:47.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.442319 539 ts_manager.cc:194] Registered new tserver with Master: 5488c47ef19a4a1583a18ae4ab2c1751 (127.0.84.194:40193)
01:52:47.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.444388 539 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.84.194:55083
01:52:47.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.444344 459 ts_manager.cc:194] Registered new tserver with Master: 5488c47ef19a4a1583a18ae4ab2c1751 (127.0.84.194:40193)
01:52:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.450461 916 heartbeater.cc:347] Connected to a master server at 127.0.84.254:33371
01:52:47.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.450860 916 heartbeater.cc:464] Registering TS with master...
01:52:47.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.451551 916 heartbeater.cc:511] Master 127.0.84.254:33371 requested a full tablet report, sending...
01:52:47.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.454077 384 ts_manager.cc:194] Registered new tserver with Master: 5488c47ef19a4a1583a18ae4ab2c1751 (127.0.84.194:40193)
01:52:47.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:47.758507 921 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:52:47.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:47.759079 921 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:52:47.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:47.759622 921 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:52:47.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:47.791502 921 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:52:47.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:47.792516 921 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.84.195
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.827580 921 tablet_server_runner.cc:78] Tablet server non-default flags:
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.84.212:33991
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-2/data
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-2/wal
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.84.195:0
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-2/data/info.pb
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.84.195
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.84.254:33371,127.0.84.253:33715,127.0.84.252:39381
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.84.195
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-2/logs
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:52:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:52:47.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.829166 921 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:52:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.830865 921 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:52:47.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:47.851670 930 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:48.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:48.449337 913 heartbeater.cc:503] Master 127.0.84.252:39381 was elected leader, sending a full tablet report...
01:52:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:47.851747 932 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:49.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:47.853603 921 server_base.cc:1034] running on GCE node
01:52:49.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:47.852577 929 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:49.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.079411 921 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:52:49.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.082181 921 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:52:49.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.083739 921 hybrid_clock.cc:648] HybridClock initialized: now 1782006769083668 us; error 77 us; skew 500 ppm
01:52:49.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.084728 921 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:52:49.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.091982 921 webserver.cc:460] Webserver started at http://127.0.84.195:45689/ using document root <none> and password file <none>
01:52:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.093184 921 fs_manager.cc:359] Metadata directory not provided
01:52:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.093439 921 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:52:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.093943 921 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:52:49.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.099402 921 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-2/data/instance:
01:52:49.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "98eb1534cea741c7861ab1b4f5d488fb"
01:52:49.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:49 on dist-test-slave-s8kj"
01:52:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.100476 921 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-2/wal/instance:
01:52:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "98eb1534cea741c7861ab1b4f5d488fb"
01:52:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:49 on dist-test-slave-s8kj"
01:52:49.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.108062 921 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
01:52:49.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.114799 940 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:52:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.116117 921 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
01:52:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.116427 921 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-2/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-2/wal
01:52:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "98eb1534cea741c7861ab1b4f5d488fb"
01:52:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:49 on dist-test-slave-s8kj"
01:52:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.116760 921 fs_report.cc:383] FS layout report
01:52:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:52:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-2/wal
01:52:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-2/wal
01:52:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-2/data/data
01:52:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:52:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:52:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:52:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:52:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:52:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:52:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:52:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:52:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:52:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:52:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:52:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.160228 921 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:52:49.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.162053 921 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:52:49.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.162467 921 kserver.cc:163] Server-wide thread pool size limit: 3276
01:52:49.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.165727 921 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:52:49.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.169893 921 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:52:49.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.170091 921 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:52:49.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.170308 921 ts_tablet_manager.cc:610] Registered 0 tablets
01:52:49.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.170506 921 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:52:49.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.322908 921 rpc_server.cc:307] RPC server started. Bound to: 127.0.84.195:44703
01:52:49.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.323081 1052 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.84.195:44703 every 8 connection(s)
01:52:49.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.325572 921 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-2/data/info.pb
01:52:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.328832 339 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 921
01:52:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.329317 339 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5505581251554135917/ts-2/wal/instance
01:52:49.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.376642 1054 heartbeater.cc:347] Connected to a master server at 127.0.84.253:33715
01:52:49.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.378826 1053 heartbeater.cc:347] Connected to a master server at 127.0.84.252:39381
01:52:49.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.379621 1053 heartbeater.cc:464] Registering TS with master...
01:52:49.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.379959 1055 heartbeater.cc:347] Connected to a master server at 127.0.84.254:33371
01:52:49.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.380259 1055 heartbeater.cc:464] Registering TS with master...
01:52:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.379623 1054 heartbeater.cc:464] Registering TS with master...
01:52:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.381368 1055 heartbeater.cc:511] Master 127.0.84.254:33371 requested a full tablet report, sending...
01:52:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.381953 1054 heartbeater.cc:511] Master 127.0.84.253:33715 requested a full tablet report, sending...
01:52:49.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.382813 1053 heartbeater.cc:511] Master 127.0.84.252:39381 requested a full tablet report, sending...
01:52:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.384879 459 ts_manager.cc:194] Registered new tserver with Master: 98eb1534cea741c7861ab1b4f5d488fb (127.0.84.195:44703)
01:52:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.385080 538 ts_manager.cc:194] Registered new tserver with Master: 98eb1534cea741c7861ab1b4f5d488fb (127.0.84.195:44703)
01:52:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.385375 384 ts_manager.cc:194] Registered new tserver with Master: 98eb1534cea741c7861ab1b4f5d488fb (127.0.84.195:44703)
01:52:49.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.387450 538 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.84.195:36597
01:52:49.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.406911 339 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
01:52:49.417 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:52:49.417 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
01:52:49.419 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.84.254"
port: 33371
}
bound_http_address {
host: "127.0.84.254"
port: 33193
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.84.253"
port: 33715
}
bound_http_address {
host: "127.0.84.253"
port: 34465
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.84.252"
port: 39381
}
bound_http_address {
host: "127.0.84.252"
port: 37387
}
}
}
01:52:49.419 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:52:49.420 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.84.193"
port: 36895
}
bound_http_address {
host: "127.0.84.193"
port: 33385
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.84.194"
port: 40193
}
bound_http_address {
host: "127.0.84.194"
port: 32877
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.84.195"
port: 44703
}
bound_http_address {
host: "127.0.84.195"
port: 45689
}
}
}
01:52:49.420 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
01:52:49.421 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster2364746489321464614/krb5kdc/krb5cc
01:52:49.421 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
01:52:49.422 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskHBGiCK/test-tmp
01:52:49.423 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:52:49.423 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskHBGiCK/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:52:49.424 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
01:52:49.766 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:52:49.766 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:52:49.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.769546 1064 mini_oidc.cc:152] Starting JWKS server
01:52:49.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.773204 1064 webserver.cc:460] Webserver started at http://127.0.0.1:37661/ using document root <none> and password file <none>
01:52:49.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.773746 1064 mini_oidc.cc:189] Starting OIDC Discovery server
01:52:49.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:49.775487 1064 webserver.cc:460] Webserver started at http://127.0.0.1:39059/ using document root <none> and password file <none>
01:52:49.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
01:52:49.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/principal' for realm 'KRBTEST.COM',
01:52:49.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
01:52:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:49 dist-test-slave-s8kj krb5kdc[1075](info): setting up network...
01:52:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
01:52:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:49 dist-test-slave-s8kj krb5kdc[1075](info): set up 2 sockets
01:52:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:49 dist-test-slave-s8kj krb5kdc[1075](info): commencing operation
01:52:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
01:52:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:50.392114 1053 heartbeater.cc:503] Master 127.0.84.252:39381 was elected leader, sending a full tablet report...
01:52:50.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:50.924170 1064 mini_kdc.cc:121] Time spent starting KDC: real 1.122s user 0.004s sys 0.015s
01:52:50.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
01:52:50.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:52:50.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
01:52:50.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
01:52:50.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:52:50.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
01:52:50.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
01:52:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:52:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
01:52:51.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:52:51.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/test-user.keytab.
01:52:51.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/test-user.keytab.
01:52:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:51 dist-test-slave-s8kj krb5kdc[1075](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006771, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:52:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:52:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:52:51Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:52:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:52:51Z Disabled control of system clock
01:52:51.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.10.62@KRBTEST.COM; defaulting to no policy
01:52:51.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:52:51.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.10.62@KRBTEST.COM" created.
01:52:51.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:52:51.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.10.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.62.keytab.
01:52:51.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.10.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.62.keytab.
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:51.126065 1064 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/wal
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/logs
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data/info.pb
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.10.62:45467
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.10.62
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.62.keytab
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.10.62
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.10.20:38093
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.10.62:45467,127.1.10.61:43181,127.1.10.60:38931
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37661/jwks.json
01:52:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:52:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:51.452142 1088 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:52:51.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:51.452760 1088 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:52:51.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:51.453588 1088 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:52:51.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:51.488116 1088 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:52:51.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:51.488391 1088 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:52:51.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:51.488687 1088 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:52:51.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:51.488886 1088 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:52:51.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:51.489167 1088 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:52:51.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:51.489377 1088 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:52:51.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:51.489517 1088 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37661/jwks.json
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:51.526974 1088 master_runner.cc:386] Master server non-default flags:
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.10.20:38093
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/wal
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.10.62:45467,127.1.10.61:43181,127.1.10.60:38931
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.62.keytab
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.10.62
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.10.62:45467
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37661/jwks.json
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data/info.pb
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.10.62
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/logs
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:52:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:51.528317 1088 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:52:51.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:51.529320 1088 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:52:51.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:51.548115 1095 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:51.548105 1094 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:51.548564 1097 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:51.549876 1088 server_base.cc:1034] running on GCE node
01:52:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.822137 1088 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:52:52.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.824682 1088 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:52:52.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.826066 1088 hybrid_clock.cc:648] HybridClock initialized: now 1782006772826021 us; error 62 us; skew 500 ppm
01:52:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.826902 1088 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:52:52.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:52 dist-test-slave-s8kj krb5kdc[1075](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006772, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:52:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.838327 1088 init.cc:377] Logged in from keytab as kudu/127.1.10.62@KRBTEST.COM (short username kudu)
01:52:52.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.844090 1088 webserver.cc:460] Webserver started at http://127.1.10.62:43447/ using document root <none> and password file <none>
01:52:52.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.845135 1088 fs_manager.cc:359] Metadata directory not provided
01:52:52.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.845373 1088 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:52:52.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.846053 1088 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:52:52.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.851302 1088 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data/instance:
01:52:52.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d0d7b1fab1b44e88a96426115d0b057"
01:52:52.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:52 on dist-test-slave-s8kj"
01:52:52.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.852559 1088 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/wal/instance:
01:52:52.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d0d7b1fab1b44e88a96426115d0b057"
01:52:52.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:52 on dist-test-slave-s8kj"
01:52:52.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.860144 1088 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.005s
01:52:52.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.865969 1106 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:52:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.867022 1088 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
01:52:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.867409 1088 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/wal
01:52:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d0d7b1fab1b44e88a96426115d0b057"
01:52:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:52 on dist-test-slave-s8kj"
01:52:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.867781 1088 fs_report.cc:383] FS layout report
01:52:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:52:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/wal
01:52:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/wal
01:52:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data/data
01:52:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:52:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:52:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:52:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:52:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:52:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:52:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:52:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:52:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:52:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:52:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:52:52.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.914139 1088 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:52:52.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.921185 1088 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:52:52.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.921707 1088 kserver.cc:163] Server-wide thread pool size limit: 3276
01:52:52.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.994534 1088 rpc_server.cc:307] RPC server started. Bound to: 127.1.10.62:45467
01:52:52.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.994660 1157 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.10.62:45467 every 8 connection(s)
01:52:52.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:52.997386 1088 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data/info.pb
01:52:53.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:53.003072 1158 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:52:53.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:53.003968 1064 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 1088
01:52:53.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:53.004765 1064 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/wal/instance
01:52:53.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:53.019424 1158 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } has no permanent_uuid. Determining permanent_uuid...
01:52:53.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.10.61@KRBTEST.COM; defaulting to no policy
01:52:53.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:52:53.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.10.61@KRBTEST.COM" created.
01:52:53.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:53 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006772, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.62@KRBTEST.COM for kudu/127.1.10.62@KRBTEST.COM
01:52:53.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:52:53.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.10.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.61.keytab.
01:52:53.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.10.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.61.keytab.
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:53.053751 1064 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/wal
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/logs
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data/info.pb
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:52:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.10.61:43181
01:52:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.10.61
01:52:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:52:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.61.keytab
01:52:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.10.61
01:52:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:52:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:52:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:52:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.10.20:38093
01:52:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:52:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:52:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.10.62:45467,127.1.10.61:43181,127.1.10.60:38931
01:52:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:52:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37661/jwks.json
01:52:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:52:53.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:53.061322 1158 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } has no permanent_uuid. Determining permanent_uuid...
01:52:53.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.063930 1109 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.10.61:43181: connect: Connection refused (error 111)
01:52:53.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.068451 1158 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.61:43181: Network error: Client connection negotiation failed: client connection to 127.1.10.61:43181: connect: Connection refused (error 111)
01:52:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:53.117981 1158 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } attempt: 1
01:52:53.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.121762 1158 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.61:43181: Network error: Client connection negotiation failed: client connection to 127.1.10.61:43181: connect: Connection refused (error 111)
01:52:53.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:53.190217 1158 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } attempt: 2
01:52:53.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.194646 1158 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.61:43181: Network error: Client connection negotiation failed: client connection to 127.1.10.61:43181: connect: Connection refused (error 111)
01:52:53.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:53.286135 1158 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } attempt: 3
01:52:53.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.290551 1158 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.61:43181: Network error: Client connection negotiation failed: client connection to 127.1.10.61:43181: connect: Connection refused (error 111)
01:52:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.371093 1164 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:52:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.371694 1164 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:52:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.372278 1164 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:52:53.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.409756 1164 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:52:53.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.410043 1164 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:52:53.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.410431 1164 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:52:53.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.410750 1164 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:52:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.411072 1164 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:52:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.411356 1164 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:52:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.411590 1164 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37661/jwks.json
01:52:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:53.434000 1158 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } attempt: 4
01:52:53.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.437652 1158 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.61:43181: Network error: Client connection negotiation failed: client connection to 127.1.10.61:43181: connect: Connection refused (error 111)
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:53.452672 1164 master_runner.cc:386] Master server non-default flags:
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.10.20:38093
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/wal
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.10.62:45467,127.1.10.61:43181,127.1.10.60:38931
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.61.keytab
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.10.61
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.10.61:43181
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37661/jwks.json
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data/info.pb
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.10.61
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/logs
01:52:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:52:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:52:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:52:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:52:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:52:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:52:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:52:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:52:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:52:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:53.454223 1164 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:52:53.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:53.455472 1164 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:52:53.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.473601 1174 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:53.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:53.737043 1158 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } attempt: 5
01:52:53.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.741775 1158 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.61:43181: Network error: Client connection negotiation failed: client connection to 127.1.10.61:43181: connect: Connection refused (error 111)
01:52:54.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.289179 1158 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } attempt: 6
01:52:54.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:54.305727 1158 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.61:43181: Network error: Client connection negotiation failed: client connection to 127.1.10.61:43181: connect: Connection refused (error 111)
01:52:54.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.474251 1171 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:54.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:53.473642 1172 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:54.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:54.717832 1173 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:52:54.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.717978 1164 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:52:54.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.721608 1164 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:52:54.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.724154 1164 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:52:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.725540 1164 hybrid_clock.cc:648] HybridClock initialized: now 1782006774725499 us; error 41 us; skew 500 ppm
01:52:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.726372 1164 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:52:54.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:54 dist-test-slave-s8kj krb5kdc[1075](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006774, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:52:54.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.732403 1164 init.cc:377] Logged in from keytab as kudu/127.1.10.61@KRBTEST.COM (short username kudu)
01:52:54.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.737855 1164 webserver.cc:460] Webserver started at http://127.1.10.61:45637/ using document root <none> and password file <none>
01:52:54.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.738826 1164 fs_manager.cc:359] Metadata directory not provided
01:52:54.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.739138 1164 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:52:54.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.739557 1164 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:52:54.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.744536 1164 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data/instance:
01:52:54.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27b29b2be3b647acb79a204cfe234081"
01:52:54.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:54 on dist-test-slave-s8kj"
01:52:54.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.745632 1164 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/wal/instance:
01:52:54.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27b29b2be3b647acb79a204cfe234081"
01:52:54.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:54 on dist-test-slave-s8kj"
01:52:54.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.753337 1164 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
01:52:54.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.758705 1184 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:52:54.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.759842 1164 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
01:52:54.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.760191 1164 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/wal
01:52:54.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27b29b2be3b647acb79a204cfe234081"
01:52:54.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:54 on dist-test-slave-s8kj"
01:52:54.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.760500 1164 fs_report.cc:383] FS layout report
01:52:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:52:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/wal
01:52:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/wal
01:52:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data/data
01:52:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:52:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:52:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:52:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:52:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:52:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:52:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:52:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:52:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:52:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:52:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:52:54.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.808696 1164 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:52:54.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.815505 1164 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:52:54.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.815961 1164 kserver.cc:163] Server-wide thread pool size limit: 3276
01:52:54.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.898836 1164 rpc_server.cc:307] RPC server started. Bound to: 127.1.10.61:43181
01:52:54.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.899017 1235 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.10.61:43181 every 8 connection(s)
01:52:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.901734 1164 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data/info.pb
01:52:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.902828 1064 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 1164
01:52:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.903265 1064 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/wal/instance
01:52:54.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.909245 1236 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:52:54.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.10.60@KRBTEST.COM; defaulting to no policy
01:52:54.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:52:54.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.10.60@KRBTEST.COM" created.
01:52:54.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.921164 1236 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } has no permanent_uuid. Determining permanent_uuid...
01:52:54.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:54 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006774, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.61@KRBTEST.COM for kudu/127.1.10.62@KRBTEST.COM
01:52:54.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:52:54.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.10.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.60.keytab.
01:52:54.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.10.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.60.keytab.
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.950611 1064 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/wal
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/logs
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data/info.pb
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:52:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:52:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.10.60:38931
01:52:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.10.60
01:52:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:52:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.60.keytab
01:52:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.10.60
01:52:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:52:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:52:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:52:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.10.20:38093
01:52:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:52:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:52:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.10.62:45467,127.1.10.61:43181,127.1.10.60:38931
01:52:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:52:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37661/jwks.json
01:52:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:52:54.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.962538 1236 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } has no permanent_uuid. Determining permanent_uuid...
01:52:54.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:54 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006774, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.61@KRBTEST.COM for kudu/127.1.10.61@KRBTEST.COM
01:52:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:54.994882 1236 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } has no permanent_uuid. Determining permanent_uuid...
01:52:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:54.997740 1188 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:52:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.005173 1236 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.60:38931: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:52:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:55.054740 1236 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } attempt: 1
01:52:55.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.058681 1236 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.60:38931: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:52:55.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:55.127115 1236 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } attempt: 2
01:52:55.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.130784 1236 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.60:38931: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:52:55.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:55.222296 1236 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } attempt: 3
01:52:55.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.226372 1236 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.60:38931: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:52:55.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.276863 1242 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:52:55.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.277441 1242 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:52:55.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.278138 1242 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:52:55.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.316978 1242 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:52:55.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.317253 1242 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:52:55.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.317523 1242 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:52:55.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.317718 1242 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:52:55.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.317903 1242 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:52:55.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.318090 1242 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:52:55.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.318231 1242 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37661/jwks.json
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:55.362783 1242 master_runner.cc:386] Master server non-default flags:
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.10.20:38093
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/wal
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.10.62:45467,127.1.10.61:43181,127.1.10.60:38931
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.60.keytab
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.10.60
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.10.60:38931
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37661/jwks.json
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:52:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data/info.pb
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.10.60
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/logs
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:52:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:55.363961 1242 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:52:55.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:55.364902 1242 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:52:55.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:55.366149 1158 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } attempt: 7
01:52:55.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:55.369837 1236 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } attempt: 4
01:52:55.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.375317 1236 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.60:38931: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:52:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:55 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006772, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.62@KRBTEST.COM for kudu/127.1.10.61@KRBTEST.COM
01:52:55.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.387629 1252 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:55.409390 1158 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } has no permanent_uuid. Determining permanent_uuid...
01:52:55.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.420504 1158 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.60:38931: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:52:55.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:55.479000 1158 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } attempt: 1
01:52:55.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.484673 1158 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.60:38931: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:52:55.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:55.566088 1158 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } attempt: 2
01:52:55.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.574029 1158 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.60:38931: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:52:55.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:55.659669 1158 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } attempt: 3
01:52:55.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.667172 1158 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.60:38931: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:52:55.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:55.674805 1236 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } attempt: 5
01:52:55.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.681490 1236 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.60:38931: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:52:55.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:55.807793 1158 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } attempt: 4
01:52:55.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.814306 1158 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.60:38931: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:52:56.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.097957 1158 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } attempt: 5
01:52:56.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:56.104297 1158 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.60:38931: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:52:56.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.229169 1236 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } attempt: 6
01:52:56.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:56.241420 1236 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.60:38931: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:52:56.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.656931 1158 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } attempt: 6
01:52:56.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:56.666571 1158 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.10.60:38931: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:52:56.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:56.784545 1251 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 1242
01:52:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:55.389434 1253 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:56.805764 1242 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.428s user 0.513s sys 0.913s
01:52:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:56.806156 1242 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.429s user 0.513s sys 0.914s
01:52:56.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:56.814539 1263 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:56.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.815789 1242 server_base.cc:1034] running on GCE node
01:52:56.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.816879 1242 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:52:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.819404 1242 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:52:56.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.820770 1242 hybrid_clock.cc:648] HybridClock initialized: now 1782006776820740 us; error 43 us; skew 500 ppm
01:52:56.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.821686 1242 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:52:56.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:56 dist-test-slave-s8kj krb5kdc[1075](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006776, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:52:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.827941 1242 init.cc:377] Logged in from keytab as kudu/127.1.10.60@KRBTEST.COM (short username kudu)
01:52:56.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.833161 1242 webserver.cc:460] Webserver started at http://127.1.10.60:41827/ using document root <none> and password file <none>
01:52:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.834194 1242 fs_manager.cc:359] Metadata directory not provided
01:52:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.834442 1242 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:52:56.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.834925 1242 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.840287 1242 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data/instance:
01:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1"
01:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:56 on dist-test-slave-s8kj"
01:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.841912 1242 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/wal/instance:
01:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1"
01:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:56 on dist-test-slave-s8kj"
01:52:56.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.850638 1242 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.002s sys 0.006s
01:52:56.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.856189 1270 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:52:56.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.857390 1242 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.857721 1242 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/wal
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1"
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:56 on dist-test-slave-s8kj"
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.858026 1242 fs_report.cc:383] FS layout report
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/wal
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/wal
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data/data
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:52:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:52:56.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.897897 1242 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:52:56.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.903795 1242 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:52:56.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.904318 1242 kserver.cc:163] Server-wide thread pool size limit: 3276
01:52:56.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.976245 1242 rpc_server.cc:307] RPC server started. Bound to: 127.1.10.60:38931
01:52:56.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.976351 1321 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.10.60:38931 every 8 connection(s)
01:52:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.979176 1242 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data/info.pb
01:52:56.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.984447 1322 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:52:56.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.985327 1064 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 1242
01:52:56.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:56.986024 1064 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/wal/instance
01:52:57.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.004585 1322 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } has no permanent_uuid. Determining permanent_uuid...
01:52:57.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:57 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006771, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.10.60@KRBTEST.COM
01:52:57.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:57 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006776, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.60@KRBTEST.COM for kudu/127.1.10.62@KRBTEST.COM
01:52:57.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.039137 1322 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } has no permanent_uuid. Determining permanent_uuid...
01:52:57.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:57 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006776, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.60@KRBTEST.COM for kudu/127.1.10.61@KRBTEST.COM
01:52:57.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:57 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006771, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.10.61@KRBTEST.COM
01:52:57.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.066004 1322 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } has no permanent_uuid. Determining permanent_uuid...
01:52:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:57 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006776, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.60@KRBTEST.COM for kudu/127.1.10.60@KRBTEST.COM
01:52:57.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:57 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006771, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.10.62@KRBTEST.COM
01:52:57.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.096612 1322 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1: Bootstrap starting.
01:52:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.102011 1064 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
01:52:57.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.102948 1322 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1: Neither blocks nor log segments found. Creating new log.
01:52:57.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.104902 1322 log.cc:824] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1: Log is configured to *not* fsync() on all Append() calls
01:52:57.109 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:52:57.109 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
01:52:57.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.109798 1322 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1: No bootstrap required, opened a new log
01:52:57.110 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
01:52:57.111 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
01:52:57.112 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.10.62"
port: 45467
}
bound_http_address {
host: "127.1.10.62"
port: 43447
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.10.61"
port: 43181
}
bound_http_address {
host: "127.1.10.61"
port: 45637
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.10.60"
port: 38931
}
bound_http_address {
host: "127.1.10.60"
port: 41827
}
}
}
01:52:57.112 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:52:57.113 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:52:57.113 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
01:52:57.114 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
01:52:57.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.133562 1322 raft_consensus.cc:348] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:52:57.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.134269 1322 raft_consensus.cc:374] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:52:57.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.134552 1322 raft_consensus.cc:729] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4fb6dd5e90564eb1a13cb9f62ce211c1, State: Initialized, Role: FOLLOWER
01:52:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.135404 1322 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:52:57.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.138327 1333 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:52:57.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.139452 1333 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [sys.catalog]: This master's current role is: FOLLOWER
01:52:57.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.139863 1322 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [sys.catalog]: configured and running, proceeding with master startup.
01:52:57.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:57 dist-test-slave-s8kj krb5kdc[1075](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006777, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:52:57.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:52:57.148 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:52:57.149 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5cc
01:52:57.151 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:52:57.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:57.159901 1276 tablet.cc:2369] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:52:57.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:57.167891 1344 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:52:57.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:57.168241 1344 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:52:57.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.163702 1242 master_runner.cc:186] Error getting master registration for 127.1.10.62:45467: OK, instance_id { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" instance_seqno: 1782006772973558 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:52:57.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.187268 1242 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:52:57.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.302132 1236 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } attempt: 7
01:52:57.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:57 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006774, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.61@KRBTEST.COM for kudu/127.1.10.60@KRBTEST.COM
01:52:57.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.340601 1333 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:52:57.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.341267 1333 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:52:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.343895 1333 leader_election.cc:288] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d0d7b1fab1b44e88a96426115d0b057 (127.1.10.62:45467), 27b29b2be3b647acb79a204cfe234081 (127.1.10.61:43181)
01:52:57.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.348415 1236 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081: Bootstrap starting.
01:52:57.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.355685 1236 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081: Neither blocks nor log segments found. Creating new log.
01:52:57.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.358297 1236 log.cc:824] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081: Log is configured to *not* fsync() on all Append() calls
01:52:57.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.363720 1236 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081: No bootstrap required, opened a new log
01:52:57.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.398413 1236 raft_consensus.cc:348] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:52:57.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.399309 1236 raft_consensus.cc:374] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:52:57.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.399622 1236 raft_consensus.cc:729] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 27b29b2be3b647acb79a204cfe234081, State: Initialized, Role: FOLLOWER
01:52:57.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.400578 1236 consensus_queue.cc:260] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:52:57.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.404083 1349 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:52:57.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.405555 1349 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: This master's current role is: FOLLOWER
01:52:57.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.407377 1211 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "27b29b2be3b647acb79a204cfe234081" is_pre_election: true
01:52:57.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.408262 1211 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4fb6dd5e90564eb1a13cb9f62ce211c1 in term 0.
01:52:57.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.409997 1273 leader_election.cc:302] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 27b29b2be3b647acb79a204cfe234081, 4fb6dd5e90564eb1a13cb9f62ce211c1; no voters:
01:52:57.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.409263 1236 sys_catalog.cc:564] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: configured and running, proceeding with master startup.
01:52:57.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.411375 1333 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [term 0 FOLLOWER]: Leader pre-election won for term 1
01:52:57.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.411777 1333 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
01:52:57.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.412294 1333 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [term 0 FOLLOWER]: Advancing to term 1
01:52:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.428174 1333 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:52:57.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.433648 1333 leader_election.cc:288] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [CANDIDATE]: Term 1 election: Requested vote from peers 3d0d7b1fab1b44e88a96426115d0b057 (127.1.10.62:45467), 27b29b2be3b647acb79a204cfe234081 (127.1.10.61:43181)
01:52:57.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.433503 1211 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "27b29b2be3b647acb79a204cfe234081"
01:52:57.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.434175 1211 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 0 FOLLOWER]: Advancing to term 1
01:52:57.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.445721 1211 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4fb6dd5e90564eb1a13cb9f62ce211c1 in term 1.
01:52:57.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.446964 1273 leader_election.cc:302] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 27b29b2be3b647acb79a204cfe234081, 4fb6dd5e90564eb1a13cb9f62ce211c1; no voters:
01:52:57.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.448220 1333 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [term 1 FOLLOWER]: Leader election won for term 1
01:52:57.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.453617 1333 raft_consensus.cc:686] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [term 1 LEADER]: Becoming Leader. State: Replica: 4fb6dd5e90564eb1a13cb9f62ce211c1, State: Running, Role: LEADER
01:52:57.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.454938 1333 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:52:57.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.469753 1357 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4fb6dd5e90564eb1a13cb9f62ce211c1. Latest consensus state: current_term: 1 leader_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:52:57.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.470695 1357 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [sys.catalog]: This master's current role is: LEADER
01:52:57.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.475605 1363 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
01:52:57.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.484241 1363 catalog_manager.cc:1484] Initializing Kudu cluster ID...
01:52:57.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.482867 1364 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
01:52:57.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.511837 1211 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 1 FOLLOWER]: Refusing update from remote peer 4fb6dd5e90564eb1a13cb9f62ce211c1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:52:57.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.513878 1357 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:52:57.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.493284 1164 master_runner.cc:186] Error getting master registration for 127.1.10.62:45467: OK, instance_id { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" instance_seqno: 1782006772973558 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:52:57.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.531272 1164 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:52:57.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.547963 1365 mvcc.cc:205] Tried to move back new op lower bound from 7299099760666025984 to 7299099760466767872. Current Snapshot: MvccSnapshot[applied={T|T < 7299099760666025984}]
01:52:57.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.550462 1333 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:52:57.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.551854 1333 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [sys.catalog]: This master's current role is: LEADER
01:52:57.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.553421 1349 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4fb6dd5e90564eb1a13cb9f62ce211c1. Latest consensus state: current_term: 1 leader_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:52:57.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.556177 1363 catalog_manager.cc:1347] Generated new cluster ID: b10bbe3fb6664081858a1e9446decf8f
01:52:57.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.556471 1363 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
01:52:57.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.557281 1349 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: This master's current role is: FOLLOWER
01:52:57.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.578799 1349 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:52:57.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.579651 1349 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: This master's current role is: FOLLOWER
01:52:57.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.594591 1363 catalog_manager.cc:1370] Generated new certificate authority record
01:52:57.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.597280 1363 catalog_manager.cc:1504] Loading token signing keys...
01:52:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.634824 1363 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1: Generated new TSK 0
01:52:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.636233 1363 catalog_manager.cc:1514] Initializing in-progress tserver states...
01:52:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.700543 1158 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } attempt: 7
01:52:57.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:57 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006772, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.62@KRBTEST.COM for kudu/127.1.10.60@KRBTEST.COM
01:52:57.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.747771 1158 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: Bootstrap starting.
01:52:57.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.756402 1158 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: Neither blocks nor log segments found. Creating new log.
01:52:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.759200 1158 log.cc:824] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: Log is configured to *not* fsync() on all Append() calls
01:52:57.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.765645 1158 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: No bootstrap required, opened a new log
01:52:57.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.798906 1158 raft_consensus.cc:348] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:52:57.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.799844 1158 raft_consensus.cc:374] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:52:57.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.800316 1158 raft_consensus.cc:729] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d0d7b1fab1b44e88a96426115d0b057, State: Initialized, Role: FOLLOWER
01:52:57.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.801355 1158 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:52:57.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.804538 1374 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:52:57.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.805469 1374 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: This master's current role is: FOLLOWER
01:52:57.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.806752 1158 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: configured and running, proceeding with master startup.
01:52:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.807356 1131 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 0 FOLLOWER]: Advancing to term 1
01:52:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.807533 1133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d0d7b1fab1b44e88a96426115d0b057" is_pre_election: true
01:52:57.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.809203 1132 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d0d7b1fab1b44e88a96426115d0b057"
01:52:57.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.815120 1131 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 1 FOLLOWER]: Refusing update from remote peer 4fb6dd5e90564eb1a13cb9f62ce211c1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:52:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.817181 1333 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:52:57.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.864522 1374 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4fb6dd5e90564eb1a13cb9f62ce211c1. Latest consensus state: current_term: 1 leader_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:52:57.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.864867 1387 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:52:57.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.865715 1387 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: This master's current role is: FOLLOWER
01:52:57.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.865697 1374 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: This master's current role is: FOLLOWER
01:52:57.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.892779 1391 catalog_manager.cc:1259] Loaded cluster ID: b10bbe3fb6664081858a1e9446decf8f
01:52:57.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.893352 1391 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: loading cluster ID for follower catalog manager: success
01:52:57.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.902410 1391 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: acquiring CA information for follower catalog manager: success
01:52:57.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:57.908808 1391 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:52:58.151 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.10.62:45467
01:52:58.152 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
01:52:58.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:58.153172 1064 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 1088
01:52:58.181 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:52:58.181 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.10.61:43181
01:52:58.181 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
01:52:58.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:58.181610 1064 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 1164
01:52:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:58.206451 1273 connection.cc:535] client connection to 127.1.10.61:43181 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
01:52:58.207 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:52:58.207 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.10.60:38931
01:52:58.207 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
01:52:58.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:58.206903 1273 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
01:52:58.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:58.207806 1064 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 1242
01:52:58.235 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:52:58.235 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.10.62:45467
01:52:58.235 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
01:52:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:58.237128 1064 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:52:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/wal
01:52:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data
01:52:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:52:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:52:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:52:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/logs
01:52:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data/info.pb
01:52:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:52:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:52:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:52:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:52:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:52:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:52:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:52:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:52:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.10.62:45467
01:52:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.10.62
01:52:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43447
01:52:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.62.keytab
01:52:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.10.62
01:52:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:52:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:52:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:52:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.10.20:38093
01:52:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:52:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:52:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.10.62:45467,127.1.10.61:43181,127.1.10.60:38931
01:52:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:52:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37661/jwks.json
01:52:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:52:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:58.563731 1396 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:52:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:58.564368 1396 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:52:58.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:58.565033 1396 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:52:58.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:58.598533 1396 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:52:58.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:58.598798 1396 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:52:58.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:58.599283 1396 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:52:58.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:58.599519 1396 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:52:58.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:58.599797 1396 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:52:58.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:58.600131 1396 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:52:58.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:58.600368 1396 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37661/jwks.json
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:58.639689 1396 master_runner.cc:386] Master server non-default flags:
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.10.20:38093
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/wal
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.10.62:45467,127.1.10.61:43181,127.1.10.60:38931
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.62.keytab
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.10.62
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.10.62:45467
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37661/jwks.json
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:52:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data/info.pb
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.10.62
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43447
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/logs
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:52:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:58.641094 1396 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:52:58.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:58.642251 1396 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:52:58.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:58.661971 1403 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:59.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:58.663257 1406 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:59.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:58.663754 1396 server_base.cc:1034] running on GCE node
01:52:59.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:52:58.662201 1404 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:52:59.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:59.916883 1396 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:52:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:59.919878 1396 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:52:59.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:59.921341 1396 hybrid_clock.cc:648] HybridClock initialized: now 1782006779921307 us; error 48 us; skew 500 ppm
01:52:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:59.922291 1396 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:52:59.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:52:59 dist-test-slave-s8kj krb5kdc[1075](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006779, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:52:59.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:59.932499 1396 init.cc:377] Logged in from keytab as kudu/127.1.10.62@KRBTEST.COM (short username kudu)
01:52:59.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:59.937426 1396 webserver.cc:460] Webserver started at http://127.1.10.62:43447/ using document root <none> and password file <none>
01:52:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:59.938501 1396 fs_manager.cc:359] Metadata directory not provided
01:52:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:59.938756 1396 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:52:59.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:59.946959 1396 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
01:52:59.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:59.953697 1416 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:52:59.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:59.955021 1396 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
01:52:59.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:59.955318 1396 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/wal
01:52:59.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d0d7b1fab1b44e88a96426115d0b057"
01:52:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:52 on dist-test-slave-s8kj"
01:52:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:52:59.957917 1396 fs_report.cc:383] FS layout report
01:52:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:52:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/wal
01:52:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/wal
01:52:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data/data
01:52:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:52:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:52:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:52:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:52:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:52:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:52:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:52:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:52:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:52:59.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:52:59.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:53:00.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.017946 1396 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:53:00.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.023988 1396 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:53:00.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.024411 1396 kserver.cc:163] Server-wide thread pool size limit: 3276
01:53:00.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.099450 1396 rpc_server.cc:307] RPC server started. Bound to: 127.1.10.62:45467
01:53:00.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.099591 1467 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.10.62:45467 every 8 connection(s)
01:53:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.102145 1396 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data/info.pb
01:53:00.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.105688 1064 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 1396
01:53:00.106 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:53:00.106 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.10.61:43181
01:53:00.106 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.108589 1064 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/wal
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/logs
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data/info.pb
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.10.61:43181
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.10.61
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45637
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.61.keytab
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.10.61
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:53:00.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.10.20:38093
01:53:00.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:53:00.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:53:00.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:53:00.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:53:00.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.10.62:45467,127.1.10.61:43181,127.1.10.60:38931
01:53:00.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:53:00.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37661/jwks.json
01:53:00.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:53:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.116516 1468 sys_catalog.cc:263] Verifying existing consensus state
01:53:00.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.122115 1468 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: Bootstrap starting.
01:53:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.159844 1468 log.cc:824] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: Log is configured to *not* fsync() on all Append() calls
01:53:00.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.175657 1468 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
01:53:00.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.176597 1468 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: Bootstrap complete.
01:53:00.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.216271 1468 raft_consensus.cc:348] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:53:00.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.220227 1468 raft_consensus.cc:729] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d0d7b1fab1b44e88a96426115d0b057, State: Initialized, Role: FOLLOWER
01:53:00.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.221377 1468 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:53:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.224802 1472 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:53:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.225837 1472 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: This master's current role is: FOLLOWER
01:53:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.226719 1468 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: configured and running, proceeding with master startup.
01:53:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.262794 1483 catalog_manager.cc:1259] Loaded cluster ID: b10bbe3fb6664081858a1e9446decf8f
01:53:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.263083 1483 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: loading cluster ID for follower catalog manager: success
01:53:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.271672 1483 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: acquiring CA information for follower catalog manager: success
01:53:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:00 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006779, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.62@KRBTEST.COM for kudu/127.1.10.62@KRBTEST.COM
01:53:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.279414 1483 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:53:00.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.305791 1396 master_runner.cc:186] Error getting master registration for 127.1.10.61:43181: Network error: Client connection negotiation failed: client connection to 127.1.10.61:43181: connect: Connection refused (error 111),
01:53:00.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.338181 1396 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:53:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:00.506858 1470 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:53:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:00.507414 1470 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:53:00.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:00.508028 1470 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:53:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:00.543368 1470 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:53:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:00.543684 1470 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:53:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:00.544061 1470 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:53:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:00.544364 1470 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:53:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:00.544651 1470 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:53:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:00.544952 1470 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:53:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:00.545208 1470 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37661/jwks.json
01:53:00.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.585767 1470 master_runner.cc:386] Master server non-default flags:
01:53:00.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:53:00.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.10.20:38093
01:53:00.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:53:00.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:53:00.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data
01:53:00.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/wal
01:53:00.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:53:00.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:53:00.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.10.62:45467,127.1.10.61:43181,127.1.10.60:38931
01:53:00.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.61.keytab
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.10.61
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.10.61:43181
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37661/jwks.json
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data/info.pb
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.10.61
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45637
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/logs
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:53:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:53:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:53:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:53:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:53:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:53:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:53:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.587275 1470 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:53:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:00.588618 1470 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:53:00.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:00.609804 1495 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:53:01.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:01.627720 1498 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:53:01.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:01.629813 1498 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:53:01.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:01.644718 1419 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.10.61:43181: connect: Connection refused (error 111)
01:53:01.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:01.648219 1498 leader_election.cc:288] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 27b29b2be3b647acb79a204cfe234081 (127.1.10.61:43181), 4fb6dd5e90564eb1a13cb9f62ce211c1 (127.1.10.60:38931)
01:53:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:01.658911 1419 leader_election.cc:334] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 27b29b2be3b647acb79a204cfe234081 (127.1.10.61:43181): Network error: Client connection negotiation failed: client connection to 127.1.10.61:43181: connect: Connection refused (error 111)
01:53:01.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:01.669203 1420 leader_election.cc:334] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 4fb6dd5e90564eb1a13cb9f62ce211c1 (127.1.10.60:38931): Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:53:01.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:01.670207 1420 leader_election.cc:302] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d0d7b1fab1b44e88a96426115d0b057; no voters: 27b29b2be3b647acb79a204cfe234081, 4fb6dd5e90564eb1a13cb9f62ce211c1
01:53:01.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:01.671660 1498 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
01:53:02.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:00.609855 1492 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:53:02.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:00.611527 1493 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:53:02.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.108587 1470 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:53:02.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:02.109556 1494 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1496 milliseconds
01:53:02.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.111224 1470 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:53:02.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.114693 1470 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:53:02.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.116232 1470 hybrid_clock.cc:648] HybridClock initialized: now 1782006782116160 us; error 70 us; skew 500 ppm
01:53:02.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.117358 1470 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:53:02.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:02 dist-test-slave-s8kj krb5kdc[1075](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006782, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:53:02.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.124686 1470 init.cc:377] Logged in from keytab as kudu/127.1.10.61@KRBTEST.COM (short username kudu)
01:53:02.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.130467 1470 webserver.cc:460] Webserver started at http://127.1.10.61:45637/ using document root <none> and password file <none>
01:53:02.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.131675 1470 fs_manager.cc:359] Metadata directory not provided
01:53:02.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.131994 1470 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:53:02.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.143114 1470 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.004s sys 0.004s
01:53:02.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.148991 1508 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:53:02.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.150422 1470 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
01:53:02.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.150820 1470 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/wal
01:53:02.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27b29b2be3b647acb79a204cfe234081"
01:53:02.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:54 on dist-test-slave-s8kj"
01:53:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.154440 1470 fs_report.cc:383] FS layout report
01:53:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:53:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/wal
01:53:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/wal
01:53:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data/data
01:53:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:53:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:53:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:53:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:53:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:53:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:53:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:53:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:53:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:53:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:53:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:53:02.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.215804 1470 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:53:02.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.223959 1470 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:53:02.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.224555 1470 kserver.cc:163] Server-wide thread pool size limit: 3276
01:53:02.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.311874 1559 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.10.61:43181 every 8 connection(s)
01:53:02.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.312304 1470 rpc_server.cc:307] RPC server started. Bound to: 127.1.10.61:43181
01:53:02.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.316267 1470 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data/info.pb
01:53:02.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.317258 1064 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 1470
01:53:02.318 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:53:02.318 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.10.60:38931
01:53:02.318 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
01:53:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.320113 1064 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:53:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:53:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/wal
01:53:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data
01:53:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:53:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:53:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:53:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:53:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/logs
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data/info.pb
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.10.60:38931
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.10.60
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41827
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.60.keytab
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.10.60
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:53:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:53:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:53:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.10.20:38093
01:53:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:53:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:53:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:53:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:53:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.10.62:45467,127.1.10.61:43181,127.1.10.60:38931
01:53:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:53:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37661/jwks.json
01:53:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:53:02.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.347065 1560 sys_catalog.cc:263] Verifying existing consensus state
01:53:02.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.353117 1560 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081: Bootstrap starting.
01:53:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.395197 1560 log.cc:824] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081: Log is configured to *not* fsync() on all Append() calls
01:53:02.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.417168 1560 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
01:53:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.418236 1560 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081: Bootstrap complete.
01:53:02.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.451974 1560 raft_consensus.cc:348] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:53:02.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.455698 1560 raft_consensus.cc:729] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 27b29b2be3b647acb79a204cfe234081, State: Initialized, Role: FOLLOWER
01:53:02.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.456818 1560 consensus_queue.cc:260] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:53:02.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.473484 1564 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:53:02.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.474444 1564 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: This master's current role is: FOLLOWER
01:53:02.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.475134 1560 sys_catalog.cc:564] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: configured and running, proceeding with master startup.
01:53:02.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.549152 1575 catalog_manager.cc:1259] Loaded cluster ID: b10bbe3fb6664081858a1e9446decf8f
01:53:02.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.551558 1575 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081: loading cluster ID for follower catalog manager: success
01:53:02.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.568156 1575 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081: acquiring CA information for follower catalog manager: success
01:53:02.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:02 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006782, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.61@KRBTEST.COM for kudu/127.1.10.62@KRBTEST.COM
01:53:02.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.600642 1575 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:53:02.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.657789 1470 master_runner.cc:186] Error getting master registration for 127.1.10.60:38931: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111),
01:53:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.706326 1470 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:53:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:02.820475 1562 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:53:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:02.821290 1562 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:53:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:02.822324 1562 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:53:02.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:02.879786 1562 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:53:02.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:02.880136 1562 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:53:02.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:02.880726 1562 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:53:02.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:02.881099 1562 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:53:02.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:02.881443 1562 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:53:02.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:02.881765 1562 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:53:02.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:02.882038 1562 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37661/jwks.json
01:53:02.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.948086 1562 master_runner.cc:386] Master server non-default flags:
01:53:02.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:53:02.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.10.20:38093
01:53:02.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:53:02.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:53:02.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data
01:53:02.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/wal
01:53:02.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:53:02.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:53:02.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.10.62:45467,127.1.10.61:43181,127.1.10.60:38931
01:53:02.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.60.keytab
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.10.60
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.10.60:38931
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37661/jwks.json
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data/info.pb
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.10.60
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41827
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/logs
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:53:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:53:02.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:53:02.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:53:02.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:53:02.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.949676 1562 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:53:02.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:02.951030 1562 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:53:02.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:02.972730 1586 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:53:03.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.558941 1591 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:53:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.561538 1591 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:53:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.574331 1591 leader_election.cc:288] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 27b29b2be3b647acb79a204cfe234081 (127.1.10.61:43181), 4fb6dd5e90564eb1a13cb9f62ce211c1 (127.1.10.60:38931)
01:53:03.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.594522 1535 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3d0d7b1fab1b44e88a96426115d0b057" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "27b29b2be3b647acb79a204cfe234081" is_pre_election: true
01:53:03.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.595654 1535 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3d0d7b1fab1b44e88a96426115d0b057 in term 1.
01:53:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.597092 1419 leader_election.cc:302] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 27b29b2be3b647acb79a204cfe234081, 3d0d7b1fab1b44e88a96426115d0b057; no voters:
01:53:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.597898 1591 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 1 FOLLOWER]: Leader pre-election won for term 2
01:53:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.598194 1591 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
01:53:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.598489 1591 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 1 FOLLOWER]: Advancing to term 2
01:53:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.607547 1591 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:53:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:03.611245 1420 leader_election.cc:334] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 4fb6dd5e90564eb1a13cb9f62ce211c1 (127.1.10.60:38931): Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:53:03.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.612489 1535 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3d0d7b1fab1b44e88a96426115d0b057" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "27b29b2be3b647acb79a204cfe234081"
01:53:03.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.613042 1535 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 1 FOLLOWER]: Advancing to term 2
01:53:03.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.622164 1535 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3d0d7b1fab1b44e88a96426115d0b057 in term 2.
01:53:03.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.624984 1591 leader_election.cc:288] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 2 election: Requested vote from peers 27b29b2be3b647acb79a204cfe234081 (127.1.10.61:43181), 4fb6dd5e90564eb1a13cb9f62ce211c1 (127.1.10.60:38931)
01:53:03.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.626446 1419 leader_election.cc:302] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 27b29b2be3b647acb79a204cfe234081, 3d0d7b1fab1b44e88a96426115d0b057; no voters:
01:53:03.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:03.629695 1420 leader_election.cc:334] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 4fb6dd5e90564eb1a13cb9f62ce211c1 (127.1.10.60:38931): Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:53:03.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.635223 1591 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 2 FOLLOWER]: Leader election won for term 2
01:53:03.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.637455 1591 raft_consensus.cc:686] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 2 LEADER]: Becoming Leader. State: Replica: 3d0d7b1fab1b44e88a96426115d0b057, State: Running, Role: LEADER
01:53:03.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.640091 1591 consensus_queue.cc:237] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:53:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.690550 1596 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3d0d7b1fab1b44e88a96426115d0b057. Latest consensus state: current_term: 2 leader_uuid: "3d0d7b1fab1b44e88a96426115d0b057" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:53:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.693248 1596 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: This master's current role is: LEADER
01:53:03.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.708287 1598 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
01:53:03.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.732615 1598 catalog_manager.cc:1484] Initializing Kudu cluster ID...
01:53:03.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.736671 1598 catalog_manager.cc:1259] Loaded cluster ID: b10bbe3fb6664081858a1e9446decf8f
01:53:03.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.736940 1598 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
01:53:03.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.754860 1598 catalog_manager.cc:1504] Loading token signing keys...
01:53:03.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.768447 1598 catalog_manager.cc:5879] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: Loaded TSK: 0
01:53:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:03.771152 1598 catalog_manager.cc:1514] Initializing in-progress tserver states...
01:53:04.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:04.085706 1535 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 2 FOLLOWER]: Refusing update from remote peer 3d0d7b1fab1b44e88a96426115d0b057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
01:53:04.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:04.087522 1596 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
01:53:04.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:04.115758 1599 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3d0d7b1fab1b44e88a96426115d0b057. Latest consensus state: current_term: 2 leader_uuid: "3d0d7b1fab1b44e88a96426115d0b057" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:53:04.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:04.116869 1599 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: This master's current role is: FOLLOWER
01:53:04.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:04.119938 1420 consensus_peers.cc:487] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 -> Peer 4fb6dd5e90564eb1a13cb9f62ce211c1 (127.1.10.60:38931): Couldn't send request to peer 4fb6dd5e90564eb1a13cb9f62ce211c1. Status: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
01:53:04.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:04.151262 1599 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "3d0d7b1fab1b44e88a96426115d0b057" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:53:04.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:04.152029 1599 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: This master's current role is: FOLLOWER
01:53:04.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:04.152724 1591 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "3d0d7b1fab1b44e88a96426115d0b057" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:53:04.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:04.153558 1591 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: This master's current role is: LEADER
01:53:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.350188 1396 master_runner.cc:186] Error getting master registration for 127.1.10.60:38931: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111),
01:53:05.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.350642 1396 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:53:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:04.368518 1584 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 1562
01:53:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:05.419346 1584 kernel_stack_watchdog.cc:198] Thread 1562 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 392ms:
01:53:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
01:53:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
01:53:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
01:53:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
01:53:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:02.973975 1585 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:53:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:05.419574 1562 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.448s user 0.591s sys 1.239s
01:53:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:05.420480 1562 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.449s user 0.591s sys 1.239s
01:53:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:05.422088 1589 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:53:05.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:05.425079 1587 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:53:05.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.425123 1562 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:53:05.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.426980 1562 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:53:05.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.430301 1562 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:53:05.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.431893 1562 hybrid_clock.cc:648] HybridClock initialized: now 1782006785431856 us; error 42 us; skew 500 ppm
01:53:05.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.433007 1562 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:53:05.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:05 dist-test-slave-s8kj krb5kdc[1075](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006785, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:53:05.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.444310 1562 init.cc:377] Logged in from keytab as kudu/127.1.10.60@KRBTEST.COM (short username kudu)
01:53:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.457873 1562 webserver.cc:460] Webserver started at http://127.1.10.60:41827/ using document root <none> and password file <none>
01:53:05.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.459625 1562 fs_manager.cc:359] Metadata directory not provided
01:53:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.459849 1562 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:53:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.468554 1562 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.001s sys 0.005s
01:53:05.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.473546 1612 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:53:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.474790 1562 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
01:53:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.475152 1562 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/wal
01:53:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1"
01:53:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:56 on dist-test-slave-s8kj"
01:53:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.477988 1562 fs_report.cc:383] FS layout report
01:53:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:53:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/wal
01:53:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/wal
01:53:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data/data
01:53:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:53:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:53:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:53:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:53:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:53:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:53:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:53:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:53:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:53:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:53:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:53:05.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.525362 1562 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:53:05.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.533385 1562 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:53:05.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.533927 1562 kserver.cc:163] Server-wide thread pool size limit: 3276
01:53:05.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.637195 1562 rpc_server.cc:307] RPC server started. Bound to: 127.1.10.60:38931
01:53:05.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.637301 1663 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.10.60:38931 every 8 connection(s)
01:53:05.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.641448 1562 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data/info.pb
01:53:05.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.649784 1064 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 1562
01:53:05.650 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:53:05.651 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.10.62:45467(127.1.10.62:45467)] connecting to peer
01:53:05.653 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.10.61:43181(127.1.10.61:43181)] connecting to peer
01:53:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.655046 1664 sys_catalog.cc:263] Verifying existing consensus state
01:53:05.657 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.10.60:38931(127.1.10.60:38931)] connecting to peer
01:53:05.659 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.10.62:45467(127.1.10.62:45467)] Successfully connected to peer
01:53:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.663031 1664 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1: Bootstrap starting.
01:53:05.666 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.10.60:38931(127.1.10.60:38931)] Successfully connected to peer
01:53:05.666 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.1.10.61:43181(127.1.10.61:43181)] Successfully connected to peer
01:53:05.676 [DEBUG - kudu-nio-1] (SecurityContext.java:225) Refreshing Kerberos credentials...
01:53:05.678 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5cc
01:53:05.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:05 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006777, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:53:05.687 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:53:05.701 [INFO - kudu-nio-1] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
01:53:05.704 [DEBUG - kudu-nio-0] (SecurityContext.java:225) Refreshing Kerberos credentials...
01:53:05.704 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5cc
01:53:05.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:05 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006777, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:53:05.713 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:53:05.714 [INFO - kudu-nio-0] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
01:53:05.719 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.10.62
01:53:05.728 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.10.61
01:53:05.739 [DEBUG - kudu-nio-0] (SecurityContext.java:225) Refreshing Kerberos credentials...
01:53:05.740 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5cc
01:53:05.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:05 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006777, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:53:05.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.746377 1664 log.cc:824] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1: Log is configured to *not* fsync() on all Append() calls
01:53:05.752 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:53:05.752 [INFO - kudu-nio-0] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
01:53:05.753 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.10.60
01:53:05.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.776681 1664 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
01:53:05.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.778188 1664 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1: Bootstrap complete.
01:53:05.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:05 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006777, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.10.61@KRBTEST.COM
01:53:05.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.812506 1664 raft_consensus.cc:348] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:53:05.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.815950 1664 raft_consensus.cc:729] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4fb6dd5e90564eb1a13cb9f62ce211c1, State: Initialized, Role: FOLLOWER
01:53:05.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.817341 1664 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:53:05.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.820683 1674 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:53:05.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.822405 1664 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [sys.catalog]: configured and running, proceeding with master startup.
01:53:05.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.822619 1674 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [sys.catalog]: This master's current role is: FOLLOWER
01:53:05.829 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x7869bb70, L:/127.0.0.1:48788 - R:/127.1.10.61:43181] using SASL/GSSAPI
01:53:05.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:05 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006777, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.10.60@KRBTEST.COM
01:53:05.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:05 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006777, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.10.62@KRBTEST.COM
01:53:05.882 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xf85597f9, L:/127.0.0.1:37632 - R:/127.1.10.60:38931] using SASL/GSSAPI
01:53:05.892 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xc04c005a, L:/127.0.0.1:38568 - R:/127.1.10.62:45467] using SASL/GSSAPI
01:53:05.893 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.10.60:38931: Catalog manager is not initialized. State: Starting
01:53:05.909 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
01:53:05.910 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.10.62:45467, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.10.61:43181, [5ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.10.60:38931, [196ms] Sub RPC ConnectToMaster: received response from server master-127.1.10.61:43181: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.1.10.60:38931: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [256ms] Sub RPC ConnectToMaster: received response from server master-127.1.10.62:45467: OK, deferred=null) after lookup
01:53:05.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.959159 1685 catalog_manager.cc:1259] Loaded cluster ID: b10bbe3fb6664081858a1e9446decf8f
01:53:05.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.959429 1685 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1: loading cluster ID for follower catalog manager: success
01:53:05.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.967731 1685 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1: acquiring CA information for follower catalog manager: success
01:53:05.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:05 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006785, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.60@KRBTEST.COM for kudu/127.1.10.62@KRBTEST.COM
01:53:05.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:05.977844 1685 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:53:06.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:06.029021 1639 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [term 1 FOLLOWER]: Advancing to term 2
01:53:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:06.060406 1691 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "3d0d7b1fab1b44e88a96426115d0b057" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:53:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:06.061616 1691 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [sys.catalog]: This master's current role is: FOLLOWER
01:53:06.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:06.067854 1674 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3d0d7b1fab1b44e88a96426115d0b057. Latest consensus state: current_term: 2 leader_uuid: "3d0d7b1fab1b44e88a96426115d0b057" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:53:06.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:06.068666 1674 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1 [sys.catalog]: This master's current role is: FOLLOWER
01:53:06.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:06.538089 1596 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "3d0d7b1fab1b44e88a96426115d0b057" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:53:06.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:06.539165 1596 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: This master's current role is: LEADER
01:53:06.914 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.10.62:45467
01:53:06.914 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
01:53:06.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:06.915134 1064 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 1396
01:53:06.945 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.10.62:45467(127.1.10.62:45467)] handling channelInactive
01:53:06.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.10.62:45467(127.1.10.62:45467)] cleaning up while in state READY due to: connection closed
01:53:06.945 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:53:06.945 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.10.61:43181
01:53:06.946 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
01:53:06.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:06.946233 1064 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 1470
01:53:06.976 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.1.10.61:43181(127.1.10.61:43181)] handling channelInactive
01:53:06.976 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:53:06.976 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.10.61:43181(127.1.10.61:43181)] cleaning up while in state READY due to: connection closed
01:53:06.976 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.10.60:38931
01:53:06.976 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
01:53:06.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:06.976827 1064 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 1562
01:53:07.002 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.10.60:38931(127.1.10.60:38931)] handling channelInactive
01:53:07.002 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:53:07.002 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.10.60:38931(127.1.10.60:38931)] cleaning up while in state READY due to: connection closed
01:53:07.002 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.10.62:45467
01:53:07.003 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
01:53:07.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:07.004346 1064 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:53:07.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:53:07.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/wal
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/logs
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data/info.pb
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.10.62:45467
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.10.62
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43447
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.62.keytab
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.10.62
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.10.20:38093
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:53:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:53:07.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:53:07.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.10.62:45467,127.1.10.61:43181,127.1.10.60:38931
01:53:07.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:53:07.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37661/jwks.json
01:53:07.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:53:07.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:07.325582 1695 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:53:07.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:07.326128 1695 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:53:07.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:07.326694 1695 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:53:07.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:07.363840 1695 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:53:07.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:07.364080 1695 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:53:07.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:07.364418 1695 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:53:07.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:07.364621 1695 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:53:07.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:07.364809 1695 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:53:07.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:07.365002 1695 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:53:07.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:07.365218 1695 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37661/jwks.json
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:07.404512 1695 master_runner.cc:386] Master server non-default flags:
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.10.20:38093
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/wal
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.10.62:45467,127.1.10.61:43181,127.1.10.60:38931
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.62.keytab
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.10.62
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.10.62:45467
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37661/jwks.json
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data/info.pb
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.10.62
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43447
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/logs
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:53:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:53:07.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:07.405838 1695 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:53:07.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:07.406863 1695 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:53:07.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:07.425184 1705 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:53:08.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:07.425271 1702 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:53:08.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:07.425264 1703 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:53:08.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:07.427482 1695 server_base.cc:1034] running on GCE node
01:53:08.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.755165 1695 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:53:08.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.758023 1695 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:53:08.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.759485 1695 hybrid_clock.cc:648] HybridClock initialized: now 1782006788759455 us; error 41 us; skew 500 ppm
01:53:08.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.760396 1695 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:53:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:08 dist-test-slave-s8kj krb5kdc[1075](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006788, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:53:08.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.766914 1695 init.cc:377] Logged in from keytab as kudu/127.1.10.62@KRBTEST.COM (short username kudu)
01:53:08.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.771706 1695 webserver.cc:460] Webserver started at http://127.1.10.62:43447/ using document root <none> and password file <none>
01:53:08.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.772722 1695 fs_manager.cc:359] Metadata directory not provided
01:53:08.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.772969 1695 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:53:08.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.781358 1695 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.001s
01:53:08.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.786870 1714 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:53:08.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.788115 1695 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
01:53:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.788414 1695 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/wal
01:53:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d0d7b1fab1b44e88a96426115d0b057"
01:53:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:52 on dist-test-slave-s8kj"
01:53:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.790910 1695 fs_report.cc:383] FS layout report
01:53:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:53:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/wal
01:53:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/wal
01:53:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data/data
01:53:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:53:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:53:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:53:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:53:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:53:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:53:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:53:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:53:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:53:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:53:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:53:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.827832 1695 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:53:08.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.834123 1695 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:53:08.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.834686 1695 kserver.cc:163] Server-wide thread pool size limit: 3276
01:53:08.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.916210 1695 rpc_server.cc:307] RPC server started. Bound to: 127.1.10.62:45467
01:53:08.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.916391 1765 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.10.62:45467 every 8 connection(s)
01:53:08.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.919148 1695 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-0/data/info.pb
01:53:08.921 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:53:08.921 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.10.61:43181
01:53:08.921 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
01:53:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.920380 1064 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 1695
01:53:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.923563 1064 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/wal
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/logs
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data/info.pb
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:53:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.10.61:43181
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.10.61
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45637
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.61.keytab
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.10.61
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.10.20:38093
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.10.62:45467,127.1.10.61:43181,127.1.10.60:38931
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37661/jwks.json
01:53:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:53:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.931520 1766 sys_catalog.cc:263] Verifying existing consensus state
01:53:08.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.936991 1766 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: Bootstrap starting.
01:53:08.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.975749 1766 log.cc:824] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: Log is configured to *not* fsync() on all Append() calls
01:53:08.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.993103 1766 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
01:53:08.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:08.993944 1766 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: Bootstrap complete.
01:53:09.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:09.018308 1766 raft_consensus.cc:348] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:53:09.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:09.020692 1766 raft_consensus.cc:729] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d0d7b1fab1b44e88a96426115d0b057, State: Initialized, Role: FOLLOWER
01:53:09.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:09.021601 1766 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:53:09.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:09.024407 1770 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:53:09.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:09.025391 1770 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: This master's current role is: FOLLOWER
01:53:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:09.025775 1766 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: configured and running, proceeding with master startup.
01:53:09.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:09.063277 1781 catalog_manager.cc:1259] Loaded cluster ID: b10bbe3fb6664081858a1e9446decf8f
01:53:09.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:09.067174 1781 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: loading cluster ID for follower catalog manager: success
01:53:09.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:09.075729 1781 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: acquiring CA information for follower catalog manager: success
01:53:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:09.081940 1781 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:53:09.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:09 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006788, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.62@KRBTEST.COM for kudu/127.1.10.62@KRBTEST.COM
01:53:09.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:09.117450 1695 master_runner.cc:186] Error getting master registration for 127.1.10.61:43181: Network error: Client connection negotiation failed: client connection to 127.1.10.61:43181: connect: Connection refused (error 111),
01:53:09.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:09.148715 1695 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:53:09.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:09.275720 1768 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:53:09.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:09.276400 1768 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:53:09.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:09.277168 1768 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:53:09.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:09.317328 1768 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:53:09.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:09.317631 1768 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:53:09.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:09.318063 1768 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:53:09.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:09.318375 1768 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:53:09.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:09.318744 1768 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:53:09.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:09.319044 1768 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:53:09.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:09.319267 1768 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37661/jwks.json
01:53:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:09.367082 1768 master_runner.cc:386] Master server non-default flags:
01:53:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:53:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.10.20:38093
01:53:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:53:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:53:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data
01:53:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/wal
01:53:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:53:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:53:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.10.62:45467,127.1.10.61:43181,127.1.10.60:38931
01:53:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.61.keytab
01:53:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.10.61
01:53:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:53:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.10.61:43181
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37661/jwks.json
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data/info.pb
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.10.61
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45637
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/logs
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:53:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:09.368650 1768 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:53:09.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:09.369902 1768 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:53:09.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:09.392781 1792 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:53:10.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:10.820468 1798 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:53:10.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:10.821084 1798 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:53:10.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:10.837718 1717 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.10.61:43181: connect: Connection refused (error 111)
01:53:10.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:10.850137 1798 leader_election.cc:288] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 27b29b2be3b647acb79a204cfe234081 (127.1.10.61:43181), 4fb6dd5e90564eb1a13cb9f62ce211c1 (127.1.10.60:38931)
01:53:10.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:10.860747 1717 leader_election.cc:334] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 27b29b2be3b647acb79a204cfe234081 (127.1.10.61:43181): Network error: Client connection negotiation failed: client connection to 127.1.10.61:43181: connect: Connection refused (error 111)
01:53:10.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:10.863276 1718 leader_election.cc:334] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 4fb6dd5e90564eb1a13cb9f62ce211c1 (127.1.10.60:38931): Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:53:10.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:10.863698 1718 leader_election.cc:302] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d0d7b1fab1b44e88a96426115d0b057; no voters: 27b29b2be3b647acb79a204cfe234081, 4fb6dd5e90564eb1a13cb9f62ce211c1
01:53:10.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:10.864472 1798 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
01:53:11.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:10.784822 1790 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 1768
01:53:11.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:11.275102 1790 kernel_stack_watchdog.cc:198] Thread 1768 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 385ms:
01:53:11.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
01:53:11.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
01:53:11.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
01:53:11.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
01:53:11.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:09.393461 1791 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:53:11.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:11.281586 1768 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.884s user 0.003s sys 0.004s
01:53:11.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:11.282027 1768 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.884s user 0.003s sys 0.004s
01:53:11.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:11.282312 1793 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1882 milliseconds
01:53:11.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:11.284207 1795 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:53:11.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.284263 1768 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:53:11.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.285709 1768 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:53:11.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.288986 1768 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:53:11.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.290536 1768 hybrid_clock.cc:648] HybridClock initialized: now 1782006791290487 us; error 57 us; skew 500 ppm
01:53:11.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.291479 1768 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:53:11.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:11 dist-test-slave-s8kj krb5kdc[1075](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006791, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:53:11.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.302690 1768 init.cc:377] Logged in from keytab as kudu/127.1.10.61@KRBTEST.COM (short username kudu)
01:53:11.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.309178 1768 webserver.cc:460] Webserver started at http://127.1.10.61:45637/ using document root <none> and password file <none>
01:53:11.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.310305 1768 fs_manager.cc:359] Metadata directory not provided
01:53:11.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.310580 1768 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:53:11.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.319196 1768 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
01:53:11.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.324604 1806 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:53:11.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.326187 1768 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
01:53:11.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.326558 1768 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/wal
01:53:11.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27b29b2be3b647acb79a204cfe234081"
01:53:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:54 on dist-test-slave-s8kj"
01:53:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.329342 1768 fs_report.cc:383] FS layout report
01:53:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:53:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/wal
01:53:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/wal
01:53:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data/data
01:53:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:53:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:53:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:53:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:53:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:53:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:53:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:53:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:53:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:53:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:53:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:53:11.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.389261 1768 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:53:11.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.396464 1768 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:53:11.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.396992 1768 kserver.cc:163] Server-wide thread pool size limit: 3276
01:53:11.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.489432 1768 rpc_server.cc:307] RPC server started. Bound to: 127.1.10.61:43181
01:53:11.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.489459 1857 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.10.61:43181 every 8 connection(s)
01:53:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.493765 1768 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-1/data/info.pb
01:53:11.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.502991 1064 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 1768
01:53:11.503 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:53:11.504 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.10.60:38931
01:53:11.504 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
01:53:11.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.506088 1064 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:53:11.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:53:11.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/wal
01:53:11.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data
01:53:11.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:53:11.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:53:11.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:53:11.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:53:11.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:53:11.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:53:11.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/logs
01:53:11.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data/info.pb
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.10.60:38931
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.10.60
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41827
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.60.keytab
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.10.60
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.10.20:38093
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:53:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.10.62:45467,127.1.10.61:43181,127.1.10.60:38931
01:53:11.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:53:11.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37661/jwks.json
01:53:11.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:53:11.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.506927 1858 sys_catalog.cc:263] Verifying existing consensus state
01:53:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.516363 1858 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081: Bootstrap starting.
01:53:11.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.570076 1858 log.cc:824] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081: Log is configured to *not* fsync() on all Append() calls
01:53:11.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.625326 1858 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
01:53:11.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.626507 1858 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081: Bootstrap complete.
01:53:11.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:11.647746 415 debug-util.cc:398] Leaking SignalData structure 0x7b08000aa080 after lost signal to thread 352
01:53:11.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:11.648530 415 debug-util.cc:398] Leaking SignalData structure 0x7b0800052320 after lost signal to thread 418
01:53:11.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.669183 1858 raft_consensus.cc:348] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:53:11.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.673851 1858 raft_consensus.cc:729] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 27b29b2be3b647acb79a204cfe234081, State: Initialized, Role: FOLLOWER
01:53:11.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.675102 1858 consensus_queue.cc:260] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:53:11.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.693830 1858 sys_catalog.cc:564] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: configured and running, proceeding with master startup.
01:53:11.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.691913 1862 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:53:11.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.704788 1862 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: This master's current role is: FOLLOWER
01:53:11.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.792296 1873 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
01:53:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:11 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006791, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.61@KRBTEST.COM for kudu/127.1.10.62@KRBTEST.COM
01:53:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:11 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006791, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.61@KRBTEST.COM for kudu/127.1.10.61@KRBTEST.COM
01:53:11.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.909469 1768 master_runner.cc:186] Error getting master registration for 127.1.10.60:38931: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111),
01:53:11.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:11.979722 1768 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:53:12.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:12.073427 1860 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:53:12.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:12.074275 1860 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:53:12.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:12.076027 1860 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:53:12.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:12.137781 1860 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:53:12.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:12.138175 1860 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:53:12.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:12.138672 1860 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:53:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:12.139091 1860 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:53:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:12.139433 1860 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:53:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:12.139744 1860 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:53:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:12.140013 1860 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37661/jwks.json
01:53:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.208621 1860 master_runner.cc:386] Master server non-default flags:
01:53:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:53:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.10.20:38093
01:53:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:53:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:53:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data
01:53:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/wal
01:53:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:53:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:53:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.10.62:45467,127.1.10.61:43181,127.1.10.60:38931
01:53:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/kudu_127.1.10.60.keytab
01:53:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.10.60
01:53:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:53:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.10.60:38931
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37661/jwks.json
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data/info.pb
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.10.60
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41827
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/logs
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.210290 1860 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:53:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.211748 1860 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:53:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:12.305933 1883 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:53:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.405875 1890 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:53:12.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.406769 1890 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:53:12.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.422456 1890 leader_election.cc:288] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 27b29b2be3b647acb79a204cfe234081 (127.1.10.61:43181), 4fb6dd5e90564eb1a13cb9f62ce211c1 (127.1.10.60:38931)
01:53:12.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:12.446955 1718 leader_election.cc:334] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 4fb6dd5e90564eb1a13cb9f62ce211c1 (127.1.10.60:38931): Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:53:12.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:12 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006788, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.62@KRBTEST.COM for kudu/127.1.10.61@KRBTEST.COM
01:53:12.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.484639 1833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3d0d7b1fab1b44e88a96426115d0b057" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "27b29b2be3b647acb79a204cfe234081" is_pre_election: true
01:53:12.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.485688 1833 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3d0d7b1fab1b44e88a96426115d0b057 in term 2.
01:53:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.489441 1717 leader_election.cc:302] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 27b29b2be3b647acb79a204cfe234081, 3d0d7b1fab1b44e88a96426115d0b057; no voters: 4fb6dd5e90564eb1a13cb9f62ce211c1
01:53:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.490283 1890 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 2 FOLLOWER]: Leader pre-election won for term 3
01:53:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.490594 1890 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
01:53:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.490897 1890 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 2 FOLLOWER]: Advancing to term 3
01:53:12.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.500897 1890 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:53:12.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.503979 1833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3d0d7b1fab1b44e88a96426115d0b057" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "27b29b2be3b647acb79a204cfe234081"
01:53:12.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.505002 1833 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 2 FOLLOWER]: Advancing to term 3
01:53:12.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.511775 1890 leader_election.cc:288] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 3 election: Requested vote from peers 27b29b2be3b647acb79a204cfe234081 (127.1.10.61:43181), 4fb6dd5e90564eb1a13cb9f62ce211c1 (127.1.10.60:38931)
01:53:12.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:12.516419 1718 leader_election.cc:334] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 4fb6dd5e90564eb1a13cb9f62ce211c1 (127.1.10.60:38931): Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111)
01:53:12.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.522902 1833 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3d0d7b1fab1b44e88a96426115d0b057 in term 3.
01:53:12.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.529290 1717 leader_election.cc:302] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 27b29b2be3b647acb79a204cfe234081, 3d0d7b1fab1b44e88a96426115d0b057; no voters: 4fb6dd5e90564eb1a13cb9f62ce211c1
01:53:12.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.530153 1890 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 3 FOLLOWER]: Leader election won for term 3
01:53:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.551316 1890 raft_consensus.cc:686] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [term 3 LEADER]: Becoming Leader. State: Replica: 3d0d7b1fab1b44e88a96426115d0b057, State: Running, Role: LEADER
01:53:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.552570 1890 consensus_queue.cc:237] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } }
01:53:12.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.575234 1894 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3d0d7b1fab1b44e88a96426115d0b057. Latest consensus state: current_term: 3 leader_uuid: "3d0d7b1fab1b44e88a96426115d0b057" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:53:12.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.576277 1894 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: This master's current role is: LEADER
01:53:12.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.588405 1896 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
01:53:12.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.609248 1896 catalog_manager.cc:1484] Initializing Kudu cluster ID...
01:53:12.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.626664 1896 catalog_manager.cc:1259] Loaded cluster ID: b10bbe3fb6664081858a1e9446decf8f
01:53:12.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.627036 1896 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
01:53:12.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.632330 1896 catalog_manager.cc:1504] Loading token signing keys...
01:53:12.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.637223 1896 catalog_manager.cc:5879] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057: Loaded TSK: 0
01:53:12.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.639096 1896 catalog_manager.cc:1514] Initializing in-progress tserver states...
01:53:12.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.804176 1873 catalog_manager.cc:1259] Loaded cluster ID: b10bbe3fb6664081858a1e9446decf8f
01:53:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.804874 1873 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081: loading cluster ID for follower catalog manager: success
01:53:12.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.820765 1873 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081: acquiring CA information for follower catalog manager: success
01:53:12.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:12.835160 1873 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:53:12.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:12.989531 1718 consensus_peers.cc:487] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 -> Peer 4fb6dd5e90564eb1a13cb9f62ce211c1 (127.1.10.60:38931): Couldn't send request to peer 4fb6dd5e90564eb1a13cb9f62ce211c1. Status: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
01:53:13.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:13.073200 1833 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [term 3 FOLLOWER]: Refusing update from remote peer 3d0d7b1fab1b44e88a96426115d0b057: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
01:53:13.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:13.075270 1894 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
01:53:13.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:13.116859 1898 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3d0d7b1fab1b44e88a96426115d0b057. Latest consensus state: current_term: 3 leader_uuid: "3d0d7b1fab1b44e88a96426115d0b057" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:53:13.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:13.134194 1898 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: This master's current role is: FOLLOWER
01:53:13.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:13.165577 1898 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "3d0d7b1fab1b44e88a96426115d0b057" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:53:13.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:13.166391 1898 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27b29b2be3b647acb79a204cfe234081 [sys.catalog]: This master's current role is: FOLLOWER
01:53:13.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:13.168907 1894 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "3d0d7b1fab1b44e88a96426115d0b057" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d0d7b1fab1b44e88a96426115d0b057" member_type: VOTER last_known_addr { host: "127.1.10.62" port: 45467 } } peers { permanent_uuid: "27b29b2be3b647acb79a204cfe234081" member_type: VOTER last_known_addr { host: "127.1.10.61" port: 43181 } } peers { permanent_uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1" member_type: VOTER last_known_addr { host: "127.1.10.60" port: 38931 } } }
01:53:13.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:13.169720 1894 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 [sys.catalog]: This master's current role is: LEADER
01:53:13.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:13.290246 909 debug-util.cc:398] Leaking SignalData structure 0x7b08000be420 after lost signal to thread 784
01:53:13.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:13.292021 909 debug-util.cc:398] Leaking SignalData structure 0x7b08000af460 after lost signal to thread 912
01:53:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:14.161149 1695 master_runner.cc:186] Error getting master registration for 127.1.10.60:38931: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111),
01:53:14.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:14.161659 1695 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:53:14.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:14.412825 1049 debug-util.cc:398] Leaking SignalData structure 0x7b08000afde0 after lost signal to thread 924
01:53:14.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:14.414036 1049 debug-util.cc:398] Leaking SignalData structure 0x7b08000afbc0 after lost signal to thread 1052
01:53:15.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:15.614763 1718 consensus_peers.cc:487] T 00000000000000000000000000000000 P 3d0d7b1fab1b44e88a96426115d0b057 -> Peer 4fb6dd5e90564eb1a13cb9f62ce211c1 (127.1.10.60:38931): Couldn't send request to peer 4fb6dd5e90564eb1a13cb9f62ce211c1. Status: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
01:53:16.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:16.066565 1718 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.10.60:38931: connect: Connection refused (error 111) [suppressed 9 similar messages]
01:53:16.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:13.694496 1882 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 1860
01:53:16.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:16.266598 1882 kernel_stack_watchdog.cc:198] Thread 1860 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 398ms:
01:53:16.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
01:53:16.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
01:53:16.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
01:53:16.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
01:53:16.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:12.306861 1884 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:53:16.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:16.268555 1860 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 3.976s user 0.939s sys 1.636s
01:53:16.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:16.269170 1860 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 3.976s user 0.939s sys 1.636s
01:53:16.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:16.269838 1886 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:53:16.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:16.271852 1885 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:53:16.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.272266 1860 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:53:16.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.273787 1860 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:53:16.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.292474 1860 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:53:16.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.293937 1860 hybrid_clock.cc:648] HybridClock initialized: now 1782006796293898 us; error 51 us; skew 500 ppm
01:53:16.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.295279 1860 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:53:16.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:16 dist-test-slave-s8kj krb5kdc[1075](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006796, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:53:16.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.304291 1860 init.cc:377] Logged in from keytab as kudu/127.1.10.60@KRBTEST.COM (short username kudu)
01:53:16.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.358106 1860 webserver.cc:460] Webserver started at http://127.1.10.60:41827/ using document root <none> and password file <none>
01:53:16.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.359635 1860 fs_manager.cc:359] Metadata directory not provided
01:53:16.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.359973 1860 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:53:16.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.378801 1860 fs_manager.cc:709] Time spent opening directory manager: real 0.013s user 0.010s sys 0.001s
01:53:16.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.391398 1919 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:53:16.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.393129 1860 fs_manager.cc:725] Time spent opening block manager: real 0.011s user 0.005s sys 0.005s
01:53:16.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.393658 1860 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/wal
01:53:16.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4fb6dd5e90564eb1a13cb9f62ce211c1"
01:53:16.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:52:56 on dist-test-slave-s8kj"
01:53:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.398562 1860 fs_report.cc:383] FS layout report
01:53:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:53:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/wal
01:53:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/wal
01:53:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data/data
01:53:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:53:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:53:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:53:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:53:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:53:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:53:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:53:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:53:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:53:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:53:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:53:16.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.491096 1860 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:53:16.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.499969 1860 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:53:16.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.500597 1860 kserver.cc:163] Server-wide thread pool size limit: 3276
01:53:16.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.945221 1970 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.10.60:38931 every 8 connection(s)
01:53:16.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.945760 1860 rpc_server.cc:307] RPC server started. Bound to: 127.1.10.60:38931
01:53:16.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.949605 1860 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/master-2/data/info.pb
01:53:16.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:16.951407 1064 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 1860
01:53:16.952 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:53:16.953 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.10.62:45467(127.1.10.62:45467)] connecting to peer
01:53:16.954 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.1.10.62:45467(127.1.10.62:45467)] Successfully connected to peer
01:53:16.977 [DEBUG - kudu-nio-1] (SecurityContext.java:225) Refreshing Kerberos credentials...
01:53:16.978 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster7007565367365605955/krb5kdc/krb5cc
01:53:16.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:16 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006777, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:53:16.983 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:53:16.983 [INFO - kudu-nio-1] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
01:53:16.984 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.10.62
01:53:17.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:17.009320 1971 sys_catalog.cc:263] Verifying existing consensus state
01:53:17.026 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.10.62:45467(127.1.10.62:45467)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
01:53:17.028 [DEBUG - main] (Connection.java:575) [peer master-127.1.10.62:45467(127.1.10.62:45467)] disconnecting while in state TERMINATED
01:53:17.028 [DEBUG - main] (Connection.java:575) [peer master-127.1.10.61:43181(127.1.10.61:43181)] disconnecting while in state TERMINATED
01:53:17.028 [DEBUG - main] (Connection.java:575) [peer master-127.1.10.60:38931(127.1.10.60:38931)] disconnecting while in state TERMINATED
01:53:17.028 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.1.10.62:45467(127.1.10.62:45467)] handling channelInactive
01:53:17.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:17.030082 1971 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1: Bootstrap starting.
01:53:17.031 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:53:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:17.030416 1972 negotiation.cc:337] Failed RPC negotiation. Trace:
01:53:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:16.955326 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:34432 (local address 127.1.10.62:45467)
01:53:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:16.976118 (+ 20792us) server_negotiation.cc:206] Beginning negotiation
01:53:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:16.976148 (+ 30us) server_negotiation.cc:399] Waiting for connection header
01:53:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:16.976209 (+ 61us) server_negotiation.cc:407] Connection header received
01:53:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:16.976357 (+ 148us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:53:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:16.976384 (+ 27us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:53:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:16.976627 (+ 243us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:53:17.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:16.976878 (+ 251us) server_negotiation.cc:226] Negotiated authn=SASL
01:53:17.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:16.988826 (+ 11948us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
01:53:17.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:16.990384 (+ 1558us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
01:53:17.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.020519 (+ 30135us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
01:53:17.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.021497 (+ 978us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
01:53:17.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.021791 (+ 294us) server_negotiation.cc:662] Negotiated TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 Kx=ECDH Au=RSA Enc=AESGCM(128) Mac=AEAD
01:53:17.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.028688 (+ 6897us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:34432: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:34432): Cannot send after transport endpoint shutdown (error 108)
01:53:17.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":20336,"thread_start_us":21834,"threads_started":1}
01:53:17.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:17.047865 1064 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 1695
01:53:17.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:17.109690 1978 negotiation.cc:337] Failed RPC negotiation. Trace:
01:53:17.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.047960 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:41972 (local address 127.1.10.60:38931)
01:53:17.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.055957 (+ 7997us) server_negotiation.cc:206] Beginning negotiation
01:53:17.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.055988 (+ 31us) server_negotiation.cc:399] Waiting for connection header
01:53:17.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.056041 (+ 53us) server_negotiation.cc:407] Connection header received
01:53:17.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.077015 (+ 20974us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:53:17.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.077093 (+ 78us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:53:17.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.077390 (+ 297us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:53:17.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.077579 (+ 189us) server_negotiation.cc:226] Negotiated authn=SASL
01:53:17.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.107766 (+ 30187us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:41972: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
01:53:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":7523,"thread_start_us":8191,"threads_started":1}
01:53:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:53:17 dist-test-slave-s8kj krb5kdc[1075](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006791, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.10.61@KRBTEST.COM for kudu/127.1.10.60@KRBTEST.COM
01:53:17.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:17.125568 1064 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 1768
01:53:17.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:17.131124 1971 log.cc:824] T 00000000000000000000000000000000 P 4fb6dd5e90564eb1a13cb9f62ce211c1: Log is configured to *not* fsync() on all Append() calls
01:53:17.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:53:17.171218 1975 negotiation.cc:337] Failed RPC negotiation. Trace:
01:53:17.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.001451 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:41966 (local address 127.1.10.60:38931)
01:53:17.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.049999 (+ 48548us) server_negotiation.cc:206] Beginning negotiation
01:53:17.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.050117 (+ 118us) server_negotiation.cc:399] Waiting for connection header
01:53:17.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.050291 (+ 174us) server_negotiation.cc:407] Connection header received
01:53:17.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.075020 (+ 24729us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:53:17.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.075086 (+ 66us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:53:17.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.075673 (+ 587us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:53:17.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.076199 (+ 526us) server_negotiation.cc:226] Negotiated authn=SASL
01:53:17.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.081618 (+ 5419us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
01:53:17.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.089088 (+ 7470us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.095049 (+ 5961us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.096751 (+ 1702us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.105593 (+ 8842us) server_negotiation.cc:657] Negotiated auth-only TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 Kx=ECDH Au=RSA Enc=AESGCM(128) Mac=AEAD
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.113670 (+ 8077us) server_negotiation.cc:365] Received SASL_INITIATE NegotiatePB request
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.113709 (+ 39us) server_negotiation.cc:881] Received SASL_INITIATE request from client
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.113778 (+ 69us) server_negotiation.cc:892] Client requested to use mechanism: GSSAPI
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.113942 (+ 164us) server_negotiation.cc:922] calling sasl_server_start()
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.123041 (+ 9099us) server_negotiation.cc:377] Sending SASL_CHALLENGE NegotiatePB response
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.124589 (+ 1548us) server_negotiation.cc:365] Received SASL_RESPONSE NegotiatePB request
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.124626 (+ 37us) server_negotiation.cc:956] Received SASL_RESPONSE request from client
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.124657 (+ 31us) server_negotiation.cc:967] calling sasl_server_step()
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.125012 (+ 355us) server_negotiation.cc:377] Sending SASL_CHALLENGE NegotiatePB response
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.125794 (+ 782us) server_negotiation.cc:365] Received SASL_RESPONSE NegotiatePB request
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.126148 (+ 354us) server_negotiation.cc:956] Received SASL_RESPONSE request from client
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.126172 (+ 24us) server_negotiation.cc:967] calling sasl_server_step()
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.127370 (+ 1198us) server_negotiation.cc:377] Sending SASL_SUCCESS NegotiatePB response
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.127745 (+ 375us) server_negotiation.cc:1035] Waiting for connection context
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:53:17.168076 (+ 40331us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:41966: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
01:53:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":46272,"thread_start_us":45732,"threads_started":1}
01:53:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:17.177928 1064 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 1860
01:53:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:53:18Z chronyd exiting
01:53:19.472 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:53:19.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:19.474130 339 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 619
01:53:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:19.503003 339 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 780
01:53:19.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:19.533465 339 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 921
01:53:19.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:19.560163 339 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 351
01:53:19.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:19.591725 339 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 421
01:53:19.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:53:19.621572 339 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 496
01:53:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:53:20Z chronyd exiting
01:53:21.811 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:974) ~[main/:?]
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845) ~[main/:?]
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: javax.security.sasl.SaslException: GSS initiate failed
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:211) ~[?:1.8.0_362]
at org.apache.kudu.client.Negotiator$1.run(Negotiator.java:959) ~[main/:?]
at org.apache.kudu.client.Negotiator$1.run(Negotiator.java:956) ~[main/:?]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_362]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_362]
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:955) ~[main/:?]
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845) ~[main/:?]
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:162) ~[?:1.8.0_362]
at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:122) ~[?:1.8.0_362]
at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:189) ~[?:1.8.0_362]
at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:224) ~[?:1.8.0_362]
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) ~[?:1.8.0_362]
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) ~[?:1.8.0_362]
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:192) ~[?:1.8.0_362]
at org.apache.kudu.client.Negotiator$1.run(Negotiator.java:959) ~[main/:?]
at org.apache.kudu.client.Negotiator$1.run(Negotiator.java:956) ~[main/:?]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_362]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_362]
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:955) ~[main/:?]
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845) ~[main/:?]
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:974)
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845)
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782)
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:211)
at org.apache.kudu.client.Negotiator$1.run(Negotiator.java:959)
at org.apache.kudu.client.Negotiator$1.run(Negotiator.java:956)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:955)
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845)
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782)
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:162)
at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:122)
at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:189)
at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:224)
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212)
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179)
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:192)
... 42 more
01:53:22.890 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: giving up after 1 attempts
01:53:22.890 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:974) ~[main/:?]
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845) ~[main/:?]
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: javax.security.sasl.SaslException: GSS initiate failed
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:211) ~[?:1.8.0_362]
at org.apache.kudu.client.Negotiator$1.run(Negotiator.java:959) ~[main/:?]
at org.apache.kudu.client.Negotiator$1.run(Negotiator.java:956) ~[main/:?]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_362]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_362]
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:955) ~[main/:?]
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845) ~[main/:?]
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:162) ~[?:1.8.0_362]
at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:122) ~[?:1.8.0_362]
at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:189) ~[?:1.8.0_362]
at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:224) ~[?:1.8.0_362]
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) ~[?:1.8.0_362]
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) ~[?:1.8.0_362]
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:192) ~[?:1.8.0_362]
at org.apache.kudu.client.Negotiator$1.run(Negotiator.java:959) ~[main/:?]
at org.apache.kudu.client.Negotiator$1.run(Negotiator.java:956) ~[main/:?]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_362]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_362]
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:955) ~[main/:?]
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845) ~[main/:?]
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:974)
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845)
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782)
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:211)
at org.apache.kudu.client.Negotiator$1.run(Negotiator.java:959)
at org.apache.kudu.client.Negotiator$1.run(Negotiator.java:956)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:955)
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845)
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782)
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:162)
at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:122)
at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:189)
at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:224)
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212)
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179)
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:192)
... 42 more