16:25:01.767 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
16:25:01.782 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskMUvRhB/test-tmp
16:25:01.789 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
16:25:01.792 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskMUvRhB/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
16:25:02.056 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
16:25:02.237 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:25:02.239 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:25:02.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:02.243675 25564 mini_oidc.cc:150] Starting JWKS server
16:25:02.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:02.247465 25564 webserver.cc:492] Webserver started at http://127.0.0.1:41465/ using document root <none> and password file <none>
16:25:02.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:02.248030 25564 mini_oidc.cc:186] Starting OIDC Discovery server
16:25:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:02.250543 25564 webserver.cc:492] Webserver started at http://127.0.0.1:38829/ using document root <none> and password file <none>
16:25:02.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-17T16:25:02Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
16:25:02.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-17T16:25:02Z Disabled control of system clock
16:25:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:02.315078 25564 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:02.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:02.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-0/wal
16:25:02.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-0/data
16:25:02.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:25:02.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:25:02.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:25:02.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:02.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:02.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-0/logs
16:25:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-0/data/info.pb
16:25:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:25:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:25:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:25:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:25:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:25:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:25:02.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:02.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:02.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.247.62:35691
16:25:02.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.247.62
16:25:02.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:02.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.247.20:37721
16:25:02.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:02.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:02.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:02.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:02.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.247.62:35691,127.24.247.61:34549,127.24.247.60:43119
16:25:02.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:02.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41465/jwks.json with env {}
16:25:02.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:02.643734 25577 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:25:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:02.644289 25577 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:25:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:02.645035 25577 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:25:02.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:02.678841 25577 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:25:02.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:02.679129 25577 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:25:02.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:02.679320 25577 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:25:02.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:02.679505 25577 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:25:02.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:02.679773 25577 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:25:02.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:02.680024 25577 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41465/jwks.json
16:25:02.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:02.717868 25577 master_runner.cc:386] Master server non-default flags:
16:25:02.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:02.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.247.20:37721
16:25:02.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:02.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:02.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-0/data
16:25:02.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-0/wal
16:25:02.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:02.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.247.62:35691,127.24.247.61:34549,127.24.247.60:43119
16:25:02.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:02.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:02.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:02.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.247.62:35691
16:25:02.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:02.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:25:02.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:02.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41465/jwks.json
16:25:02.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:02.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:02.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-0/data/info.pb
16:25:02.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.247.62
16:25:02.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:25:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:25:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:25:02.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:02.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-0/logs
16:25:02.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:02.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:25:02.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:25:02.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:25:02.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:25:02.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:25:02.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:25:02.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:25:02.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:25:02.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:25:02.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:02.719601 25577 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:25:02.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:02.721877 25577 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:25:02.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:02.736244 25583 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:03.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:02.742789 25584 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:03.988626 25585 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
16:25:03.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:03.990198 25586 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:03.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:03.990218 25577 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:25:03.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:03.993794 25577 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:25:03.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:03.997802 25577 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:25:04.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.001488 25577 hybrid_clock.cc:648] HybridClock initialized: now 1773764704001371 us; error 120 us; skew 500 ppm
16:25:04.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.002569 25577 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:25:04.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.010097 25577 webserver.cc:492] Webserver started at http://127.24.247.62:33409/ using document root <none> and password file <none>
16:25:04.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.011193 25577 fs_manager.cc:362] Metadata directory not provided
16:25:04.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.011399 25577 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:25:04.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.012002 25577 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:25:04.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.016786 25577 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-0/data/instance:
16:25:04.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2963670866dc423e9c6b53688132fc56"
16:25:04.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:04 on dist-test-slave-jdhl"
16:25:04.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.018174 25577 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-0/wal/instance:
16:25:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2963670866dc423e9c6b53688132fc56"
16:25:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:04 on dist-test-slave-jdhl"
16:25:04.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.026808 25577 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
16:25:04.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.033205 25593 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:25:04.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.034696 25577 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
16:25:04.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.035238 25577 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-0/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-0/wal
16:25:04.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2963670866dc423e9c6b53688132fc56"
16:25:04.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:04 on dist-test-slave-jdhl"
16:25:04.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.035686 25577 fs_report.cc:389] FS layout report
16:25:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:25:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-0/wal
16:25:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-0/wal
16:25:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-0/data/data
16:25:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:25:04.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:25:04.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:25:04.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:25:04.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:25:04.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:25:04.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:25:04.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:25:04.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:25:04.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:25:04.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:25:04.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.102272 25577 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:25:04.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.104059 25577 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:25:04.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.104642 25577 kserver.cc:163] Server-wide thread pool size limit: 3276
16:25:04.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.182714 25577 rpc_server.cc:307] RPC server started. Bound to: 127.24.247.62:35691
16:25:04.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.182911 25644 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.247.62:35691 every 8 connection(s)
16:25:04.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.185858 25577 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-0/data/info.pb
16:25:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.188392 25564 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 25577
16:25:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.189064 25564 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-0/wal/instance
16:25:04.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.195190 25564 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:04.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:04.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-1/wal
16:25:04.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-1/data
16:25:04.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:25:04.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:25:04.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:25:04.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:04.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-1/logs
16:25:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-1/data/info.pb
16:25:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:04.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:25:04.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:25:04.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:25:04.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:25:04.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:04.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:04.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:04.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:25:04.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:25:04.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:04.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.247.61:34549
16:25:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.247.61
16:25:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.247.20:37721
16:25:04.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:04.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:04.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:04.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:04.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.247.62:35691,127.24.247.61:34549,127.24.247.60:43119
16:25:04.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:04.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41465/jwks.json with env {}
16:25:04.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.195539 25645 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:25:04.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.212677 25645 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } has no permanent_uuid. Determining permanent_uuid...
16:25:04.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.242396 25645 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } has no permanent_uuid. Determining permanent_uuid...
16:25:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:04.245000 25594 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.247.61:34549: connect: Connection refused (error 111)
16:25:04.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:04.249980 25645 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.61:34549: Network error: Client connection negotiation failed: client connection to 127.24.247.61:34549: connect: Connection refused (error 111)
16:25:04.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.299582 25645 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } attempt: 1
16:25:04.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:04.304620 25645 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.61:34549: Network error: Client connection negotiation failed: client connection to 127.24.247.61:34549: connect: Connection refused (error 111)
16:25:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.373047 25645 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } attempt: 2
16:25:04.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:04.378499 25645 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.61:34549: Network error: Client connection negotiation failed: client connection to 127.24.247.61:34549: connect: Connection refused (error 111)
16:25:04.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.470053 25645 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } attempt: 3
16:25:04.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:04.474664 25645 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.61:34549: Network error: Client connection negotiation failed: client connection to 127.24.247.61:34549: connect: Connection refused (error 111)
16:25:04.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:04.537226 25647 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:25:04.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:04.537910 25647 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:25:04.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:04.538388 25647 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:25:04.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:04.570986 25647 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:25:04.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:04.571283 25647 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:25:04.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:04.571587 25647 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:25:04.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:04.571902 25647 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:25:04.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:04.572223 25647 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:25:04.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:04.572474 25647 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41465/jwks.json
16:25:04.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.618222 25645 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } attempt: 4
16:25:04.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.618294 25647 master_runner.cc:386] Master server non-default flags:
16:25:04.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:04.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.247.20:37721
16:25:04.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:04.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:04.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-1/data
16:25:04.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-1/wal
16:25:04.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:04.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.247.62:35691,127.24.247.61:34549,127.24.247.60:43119
16:25:04.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:04.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:04.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.247.61:34549
16:25:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:25:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41465/jwks.json
16:25:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:04.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:04.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-1/data/info.pb
16:25:04.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.247.61
16:25:04.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:04.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:25:04.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:04.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:25:04.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:25:04.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:04.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-1/logs
16:25:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:25:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:25:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:25:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:25:04.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:25:04.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:25:04.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:25:04.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:25:04.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:25:04.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.619971 25647 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:25:04.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.622460 25647 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:25:04.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:04.623505 25645 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.61:34549: Network error: Client connection negotiation failed: client connection to 127.24.247.61:34549: connect: Connection refused (error 111)
16:25:04.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:04.636610 25656 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:04.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:04.922953 25645 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } attempt: 5
16:25:04.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:04.933255 25645 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.61:34549: Network error: Client connection negotiation failed: client connection to 127.24.247.61:34549: connect: Connection refused (error 111)
16:25:05.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:05.480907 25645 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } attempt: 6
16:25:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:05.511284 25645 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.61:34549: Network error: Client connection negotiation failed: client connection to 127.24.247.61:34549: connect: Connection refused (error 111)
16:25:06.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.041311 25655 debug-util.cc:398] Leaking SignalData structure 0x7b080002edc0 after lost signal to thread 25647
16:25:06.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.154922 25655 kernel_stack_watchdog.cc:198] Thread 25647 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
16:25:06.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:25:06.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:25:06.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:25:06.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:25:06.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:04.638350 25657 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:06.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.156409 25658 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1518 milliseconds
16:25:06.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.156838 25647 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.520s user 0.000s sys 0.002s
16:25:06.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.157176 25647 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.520s user 0.000s sys 0.002s
16:25:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.158299 25647 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:25:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.158371 25659 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:06.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.161322 25647 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:25:06.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.163992 25647 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:25:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.165419 25647 hybrid_clock.cc:648] HybridClock initialized: now 1773764706165382 us; error 36 us; skew 500 ppm
16:25:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.166441 25647 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:25:06.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.172894 25647 webserver.cc:492] Webserver started at http://127.24.247.61:33079/ using document root <none> and password file <none>
16:25:06.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.174054 25647 fs_manager.cc:362] Metadata directory not provided
16:25:06.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.174324 25647 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:25:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.174872 25647 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:25:06.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.179452 25647 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-1/data/instance:
16:25:06.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c0336dff3fa14b4a829afda4fbacce67"
16:25:06.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:06 on dist-test-slave-jdhl"
16:25:06.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.181144 25647 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-1/wal/instance:
16:25:06.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c0336dff3fa14b4a829afda4fbacce67"
16:25:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:06 on dist-test-slave-jdhl"
16:25:06.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.190619 25647 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.008s sys 0.000s
16:25:06.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.196955 25668 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:25:06.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.198622 25647 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
16:25:06.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.199046 25647 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-1/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-1/wal
16:25:06.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c0336dff3fa14b4a829afda4fbacce67"
16:25:06.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:06 on dist-test-slave-jdhl"
16:25:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.199510 25647 fs_report.cc:389] FS layout report
16:25:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:25:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-1/wal
16:25:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-1/wal
16:25:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-1/data/data
16:25:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:25:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:25:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:25:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:25:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:25:06.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:25:06.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:25:06.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:25:06.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:25:06.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:25:06.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:25:06.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.263957 25647 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:25:06.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.265749 25647 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:25:06.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.266319 25647 kserver.cc:163] Server-wide thread pool size limit: 3276
16:25:06.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.340435 25647 rpc_server.cc:307] RPC server started. Bound to: 127.24.247.61:34549
16:25:06.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.340538 25719 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.247.61:34549 every 8 connection(s)
16:25:06.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.343482 25647 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-1/data/info.pb
16:25:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.350682 25720 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:25:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.352159 25564 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 25647
16:25:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.352574 25564 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-1/wal/instance
16:25:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.354934 25564 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-2/wal
16:25:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-2/data
16:25:06.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:25:06.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:25:06.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:25:06.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:06.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:06.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:06.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-2/logs
16:25:06.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-2/data/info.pb
16:25:06.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:06.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:25:06.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:25:06.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:25:06.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:25:06.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:06.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:06.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:06.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:25:06.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:25:06.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:06.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:06.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.247.60:43119
16:25:06.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.247.60
16:25:06.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:06.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.247.20:37721
16:25:06.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:06.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:06.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:06.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:06.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.247.62:35691,127.24.247.61:34549,127.24.247.60:43119
16:25:06.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:06.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41465/jwks.json with env {}
16:25:06.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.364332 25720 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } has no permanent_uuid. Determining permanent_uuid...
16:25:06.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.395097 25720 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } has no permanent_uuid. Determining permanent_uuid...
16:25:06.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.409855 25720 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } has no permanent_uuid. Determining permanent_uuid...
16:25:06.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.412981 25669 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.247.60:43119: connect: Connection refused (error 111)
16:25:06.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.418390 25720 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.60:43119: Network error: Client connection negotiation failed: client connection to 127.24.247.60:43119: connect: Connection refused (error 111)
16:25:06.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.467938 25720 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } attempt: 1
16:25:06.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.473246 25720 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.60:43119: Network error: Client connection negotiation failed: client connection to 127.24.247.60:43119: connect: Connection refused (error 111)
16:25:06.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.541707 25720 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } attempt: 2
16:25:06.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.546377 25720 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.60:43119: Network error: Client connection negotiation failed: client connection to 127.24.247.60:43119: connect: Connection refused (error 111)
16:25:06.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.571887 25645 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } attempt: 7
16:25:06.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.585772 25645 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } has no permanent_uuid. Determining permanent_uuid...
16:25:06.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.592339 25645 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.60:43119: Network error: Client connection negotiation failed: client connection to 127.24.247.60:43119: connect: Connection refused (error 111)
16:25:06.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.637873 25720 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } attempt: 3
16:25:06.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.642305 25720 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.60:43119: Network error: Client connection negotiation failed: client connection to 127.24.247.60:43119: connect: Connection refused (error 111)
16:25:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.650923 25645 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } attempt: 1
16:25:06.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.655454 25645 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.60:43119: Network error: Client connection negotiation failed: client connection to 127.24.247.60:43119: connect: Connection refused (error 111)
16:25:06.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.697260 25722 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:25:06.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.697966 25722 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:25:06.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.698675 25722 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:25:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.731685 25722 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:25:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.732086 25722 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:25:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.732414 25722 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:25:06.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.732824 25722 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:25:06.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.733186 25722 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:25:06.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.733477 25722 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41465/jwks.json
16:25:06.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.736945 25645 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } attempt: 2
16:25:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.741021 25645 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.60:43119: Network error: Client connection negotiation failed: client connection to 127.24.247.60:43119: connect: Connection refused (error 111)
16:25:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.774674 25722 master_runner.cc:386] Master server non-default flags:
16:25:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.247.20:37721
16:25:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-2/data
16:25:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-2/wal
16:25:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.247.62:35691,127.24.247.61:34549,127.24.247.60:43119
16:25:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.247.60:43119
16:25:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:25:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41465/jwks.json
16:25:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-2/data/info.pb
16:25:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.247.60
16:25:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:25:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:25:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:25:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-2/logs
16:25:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:25:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:25:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:25:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:25:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:25:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:25:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:25:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:25:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:25:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.776108 25722 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:25:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.778316 25722 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:25:06.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.785737 25720 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } attempt: 4
16:25:06.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.791011 25734 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:06.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.791687 25720 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.60:43119: Network error: Client connection negotiation failed: client connection to 127.24.247.60:43119: connect: Connection refused (error 111)
16:25:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.826490 25645 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } attempt: 3
16:25:06.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.835361 25645 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.60:43119: Network error: Client connection negotiation failed: client connection to 127.24.247.60:43119: connect: Connection refused (error 111)
16:25:06.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:06.975991 25645 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } attempt: 4
16:25:06.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.986479 25645 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.60:43119: Network error: Client connection negotiation failed: client connection to 127.24.247.60:43119: connect: Connection refused (error 111)
16:25:07.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:07.091217 25720 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } attempt: 5
16:25:07.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:07.100471 25720 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.60:43119: Network error: Client connection negotiation failed: client connection to 127.24.247.60:43119: connect: Connection refused (error 111)
16:25:07.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:07.270052 25645 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } attempt: 5
16:25:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:07.278627 25645 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.60:43119: Network error: Client connection negotiation failed: client connection to 127.24.247.60:43119: connect: Connection refused (error 111)
16:25:07.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:07.648151 25720 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } attempt: 6
16:25:07.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:07.686815 25720 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.60:43119: Network error: Client connection negotiation failed: client connection to 127.24.247.60:43119: connect: Connection refused (error 111)
16:25:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:07.831219 25645 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } attempt: 6
16:25:07.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:07.859169 25645 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.247.60:43119: Network error: Client connection negotiation failed: client connection to 127.24.247.60:43119: connect: Connection refused (error 111)
16:25:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:08.195168 25733 debug-util.cc:398] Leaking SignalData structure 0x7b080002edc0 after lost signal to thread 25722
16:25:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:08.528403 25733 kernel_stack_watchdog.cc:198] Thread 25722 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
16:25:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:25:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:25:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:25:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:25:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:08.530097 25722 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.738s user 0.625s sys 1.085s
16:25:08.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:08.530733 25736 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1736 milliseconds
16:25:08.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:06.794482 25735 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:08.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:08.531677 25737 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:08.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:08.530972 25722 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.739s user 0.626s sys 1.085s
16:25:08.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.532930 25722 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:25:08.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.537752 25722 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:25:08.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.540824 25722 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:25:08.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.542246 25722 hybrid_clock.cc:648] HybridClock initialized: now 1773764708542187 us; error 69 us; skew 500 ppm
16:25:08.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.543670 25722 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:25:08.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.551151 25722 webserver.cc:492] Webserver started at http://127.24.247.60:41809/ using document root <none> and password file <none>
16:25:08.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.552464 25722 fs_manager.cc:362] Metadata directory not provided
16:25:08.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.552809 25722 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:25:08.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.553455 25722 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:25:08.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.558327 25722 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-2/data/instance:
16:25:08.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d"
16:25:08.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:08 on dist-test-slave-jdhl"
16:25:08.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.559688 25722 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-2/wal/instance:
16:25:08.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d"
16:25:08.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:08 on dist-test-slave-jdhl"
16:25:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.568610 25722 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.002s sys 0.006s
16:25:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.577627 25748 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:25:08.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.579105 25722 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.006s sys 0.000s
16:25:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.579561 25722 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-2/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-2/wal
16:25:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d"
16:25:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:08 on dist-test-slave-jdhl"
16:25:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.580076 25722 fs_report.cc:389] FS layout report
16:25:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:25:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-2/wal
16:25:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-2/wal
16:25:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-2/data/data
16:25:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:25:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:25:08.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:25:08.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:25:08.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:25:08.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:25:08.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:25:08.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:25:08.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:25:08.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:25:08.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:25:08.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.637164 25722 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:25:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.639075 25722 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:25:08.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.639649 25722 kserver.cc:163] Server-wide thread pool size limit: 3276
16:25:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.719256 25722 rpc_server.cc:307] RPC server started. Bound to: 127.24.247.60:43119
16:25:08.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.719312 25799 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.247.60:43119 every 8 connection(s)
16:25:08.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.722537 25722 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-2/data/info.pb
16:25:08.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.730501 25564 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 25722
16:25:08.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.731101 25564 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/master-2/wal/instance
16:25:08.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.731312 25800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:25:08.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.747411 25720 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } attempt: 7
16:25:08.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.758723 25800 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } has no permanent_uuid. Determining permanent_uuid...
16:25:08.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.799307 25720 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67: Bootstrap starting.
16:25:08.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.800412 25800 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } has no permanent_uuid. Determining permanent_uuid...
16:25:08.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.811146 25720 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67: Neither blocks nor log segments found. Creating new log.
16:25:08.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.812824 25800 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } has no permanent_uuid. Determining permanent_uuid...
16:25:08.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.813477 25720 log.cc:826] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67: Log is configured to *not* fsync() on all Append() calls
16:25:08.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.822856 25720 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67: No bootstrap required, opened a new log
16:25:08.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.832660 25800 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d: Bootstrap starting.
16:25:08.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.841008 25800 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d: Neither blocks nor log segments found. Creating new log.
16:25:08.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.843104 25800 log.cc:826] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d: Log is configured to *not* fsync() on all Append() calls
16:25:08.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.846449 25720 raft_consensus.cc:359] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } }
16:25:08.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.848083 25720 raft_consensus.cc:385] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:25:08.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.848809 25720 raft_consensus.cc:740] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c0336dff3fa14b4a829afda4fbacce67, State: Initialized, Role: FOLLOWER
16:25:08.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.849225 25800 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d: No bootstrap required, opened a new log
16:25:08.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.850286 25720 consensus_queue.cc:260] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } }
16:25:08.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.853415 25811 sys_catalog.cc:455] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } } }
16:25:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.854287 25811 sys_catalog.cc:458] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67 [sys.catalog]: This master's current role is: FOLLOWER
16:25:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.855396 25720 sys_catalog.cc:565] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67 [sys.catalog]: configured and running, proceeding with master startup.
16:25:08.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.874931 25800 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } }
16:25:08.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.875811 25800 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:25:08.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.876111 25800 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9b5aeb6b1dd84c09bbfc66c8d559722d, State: Initialized, Role: FOLLOWER
16:25:08.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.877194 25800 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } }
16:25:08.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.881090 25821 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } } }
16:25:08.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.882512 25821 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d [sys.catalog]: This master's current role is: FOLLOWER
16:25:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.883445 25800 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d [sys.catalog]: configured and running, proceeding with master startup.
16:25:08.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.892769 25645 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } attempt: 7
16:25:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:08.895120 25754 tablet.cc:2379] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:25:08.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:08.906663 25823 catalog_manager.cc:1568] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:25:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:08.908663 25823 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:25:08.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.891295 25647 master_runner.cc:186] Error getting master registration for 127.24.247.62:35691: OK, instance_id { permanent_uuid: "2963670866dc423e9c6b53688132fc56" instance_seqno: 1773764704160613 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:25:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.931420 25647 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:25:08.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.940964 25645 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56: Bootstrap starting.
16:25:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.951277 25645 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56: Neither blocks nor log segments found. Creating new log.
16:25:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.954377 25645 log.cc:826] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56: Log is configured to *not* fsync() on all Append() calls
16:25:08.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.975716 25645 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56: No bootstrap required, opened a new log
16:25:08.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.960316 25722 master_runner.cc:186] Error getting master registration for 127.24.247.62:35691: OK, instance_id { permanent_uuid: "2963670866dc423e9c6b53688132fc56" instance_seqno: 1773764704160613 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:25:08.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:08.999179 25722 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:25:09.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:09.006507 25836 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:25:09.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:09.007632 25836 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:25:09.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.008050 25645 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } }
16:25:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.010161 25645 raft_consensus.cc:385] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:25:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.011124 25645 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2963670866dc423e9c6b53688132fc56, State: Initialized, Role: FOLLOWER
16:25:09.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.012619 25645 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } }
16:25:09.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.016130 25839 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } } }
16:25:09.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.017553 25839 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [sys.catalog]: This master's current role is: FOLLOWER
16:25:09.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.019270 25645 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [sys.catalog]: configured and running, proceeding with master startup.
16:25:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:09.025485 25674 tablet.cc:2379] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:25:09.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.037701 25839 raft_consensus.cc:493] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:25:09.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.038305 25839 raft_consensus.cc:515] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } }
16:25:09.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.041102 25839 leader_election.cc:290] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c0336dff3fa14b4a829afda4fbacce67 (127.24.247.61:34549), 9b5aeb6b1dd84c09bbfc66c8d559722d (127.24.247.60:43119)
16:25:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.043792 25695 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2963670866dc423e9c6b53688132fc56" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c0336dff3fa14b4a829afda4fbacce67" is_pre_election: true
16:25:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.044783 25695 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2963670866dc423e9c6b53688132fc56 in term 0.
16:25:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.046274 25775 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2963670866dc423e9c6b53688132fc56" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" is_pre_election: true
16:25:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.047307 25775 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2963670866dc423e9c6b53688132fc56 in term 0.
16:25:09.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.050005 25594 leader_election.cc:304] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2963670866dc423e9c6b53688132fc56, c0336dff3fa14b4a829afda4fbacce67; no voters:
16:25:09.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.051738 25839 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [term 0 FOLLOWER]: Leader pre-election won for term 1
16:25:09.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.052138 25839 raft_consensus.cc:493] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:25:09.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.052465 25839 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [term 0 FOLLOWER]: Advancing to term 1
16:25:09.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.073405 25839 raft_consensus.cc:515] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } }
16:25:09.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.085222 25695 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2963670866dc423e9c6b53688132fc56" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c0336dff3fa14b4a829afda4fbacce67"
16:25:09.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.085846 25695 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67 [term 0 FOLLOWER]: Advancing to term 1
16:25:09.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.095621 25695 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2963670866dc423e9c6b53688132fc56 in term 1.
16:25:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.106273 25775 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2963670866dc423e9c6b53688132fc56" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d"
16:25:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.107241 25775 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d [term 0 FOLLOWER]: Advancing to term 1
16:25:09.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:09.116766 25599 tablet.cc:2379] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:25:09.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.117944 25839 leader_election.cc:290] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [CANDIDATE]: Term 1 election: Requested vote from peers c0336dff3fa14b4a829afda4fbacce67 (127.24.247.61:34549), 9b5aeb6b1dd84c09bbfc66c8d559722d (127.24.247.60:43119)
16:25:09.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.128571 25594 leader_election.cc:304] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2963670866dc423e9c6b53688132fc56, c0336dff3fa14b4a829afda4fbacce67; no voters:
16:25:09.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.134310 25839 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [term 1 FOLLOWER]: Leader election won for term 1
16:25:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.136217 25839 raft_consensus.cc:697] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [term 1 LEADER]: Becoming Leader. State: Replica: 2963670866dc423e9c6b53688132fc56, State: Running, Role: LEADER
16:25:09.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.116844 25775 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2963670866dc423e9c6b53688132fc56 in term 1.
16:25:09.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.140278 25839 consensus_queue.cc:237] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } }
16:25:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.171330 25847 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2963670866dc423e9c6b53688132fc56. Latest consensus state: current_term: 1 leader_uuid: "2963670866dc423e9c6b53688132fc56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } } }
16:25:09.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.173122 25847 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [sys.catalog]: This master's current role is: LEADER
16:25:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.193342 25852 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
16:25:09.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.223237 25852 catalog_manager.cc:1494] Initializing Kudu cluster ID...
16:25:09.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.246615 25854 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
16:25:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.291554 25775 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d [term 1 FOLLOWER]: Refusing update from remote peer 2963670866dc423e9c6b53688132fc56: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:25:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.293561 25695 raft_consensus.cc:1275] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67 [term 1 FOLLOWER]: Refusing update from remote peer 2963670866dc423e9c6b53688132fc56: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:25:09.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.295215 25847 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:25:09.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.298188 25847 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:25:09.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.340716 25811 sys_catalog.cc:455] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2963670866dc423e9c6b53688132fc56. Latest consensus state: current_term: 1 leader_uuid: "2963670866dc423e9c6b53688132fc56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } } }
16:25:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.341439 25811 sys_catalog.cc:458] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67 [sys.catalog]: This master's current role is: FOLLOWER
16:25:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.344312 25821 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2963670866dc423e9c6b53688132fc56. Latest consensus state: current_term: 1 leader_uuid: "2963670866dc423e9c6b53688132fc56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } } }
16:25:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.345069 25821 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d [sys.catalog]: This master's current role is: FOLLOWER
16:25:09.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.354509 25855 mvcc.cc:204] Tried to move back new op lower bound from 7265340249237667840 to 7265340248705789952. Current Snapshot: MvccSnapshot[applied={T|T < 7265340249237667840}]
16:25:09.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.360152 25856 mvcc.cc:204] Tried to move back new op lower bound from 7265340249237667840 to 7265340248705789952. Current Snapshot: MvccSnapshot[applied={T|T < 7265340249237667840}]
16:25:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.367321 25839 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2963670866dc423e9c6b53688132fc56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } } }
16:25:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.368098 25839 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [sys.catalog]: This master's current role is: LEADER
16:25:09.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.372188 25821 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2963670866dc423e9c6b53688132fc56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } } }
16:25:09.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.372908 25821 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d [sys.catalog]: This master's current role is: FOLLOWER
16:25:09.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.378155 25852 catalog_manager.cc:1357] Generated new cluster ID: 353bedbeafb147a5bb2aac576060b9ff
16:25:09.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.378492 25852 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
16:25:09.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.387032 25857 mvcc.cc:204] Tried to move back new op lower bound from 7265340249237667840 to 7265340248705789952. Current Snapshot: MvccSnapshot[applied={T|T < 7265340249237667840}]
16:25:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.388159 25839 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2963670866dc423e9c6b53688132fc56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } } }
16:25:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.388810 25839 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56 [sys.catalog]: This master's current role is: LEADER
16:25:09.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.409834 25811 sys_catalog.cc:455] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2963670866dc423e9c6b53688132fc56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2963670866dc423e9c6b53688132fc56" member_type: VOTER last_known_addr { host: "127.24.247.62" port: 35691 } } peers { permanent_uuid: "c0336dff3fa14b4a829afda4fbacce67" member_type: VOTER last_known_addr { host: "127.24.247.61" port: 34549 } } peers { permanent_uuid: "9b5aeb6b1dd84c09bbfc66c8d559722d" member_type: VOTER last_known_addr { host: "127.24.247.60" port: 43119 } } }
16:25:09.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.410597 25811 sys_catalog.cc:458] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67 [sys.catalog]: This master's current role is: FOLLOWER
16:25:09.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.436703 25852 catalog_manager.cc:1380] Generated new certificate authority record
16:25:09.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.439877 25852 catalog_manager.cc:1514] Loading token signing keys...
16:25:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.544780 25852 catalog_manager.cc:6044] T 00000000000000000000000000000000 P 2963670866dc423e9c6b53688132fc56: Generated new TSK 0
16:25:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.546533 25852 catalog_manager.cc:1524] Initializing in-progress tserver states...
16:25:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.586669 25564 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-0/wal
16:25:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-0/data
16:25:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:25:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:25:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:25:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-0/logs
16:25:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-0/data/info.pb
16:25:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:25:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:25:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:25:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:25:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:25:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:25:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.247.1:0
16:25:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.247.1
16:25:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.247.1
16:25:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.247.62:35691,127.24.247.61:34549,127.24.247.60:43119
16:25:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.247.20:37721
16:25:09.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:09.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:09.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:25:09.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.919674 25823 catalog_manager.cc:1269] Loaded cluster ID: 353bedbeafb147a5bb2aac576060b9ff
16:25:09.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.920674 25823 catalog_manager.cc:1562] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67: loading cluster ID for follower catalog manager: success
16:25:09.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.928954 25823 catalog_manager.cc:1584] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67: acquiring CA information for follower catalog manager: success
16:25:09.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:09.944589 25823 catalog_manager.cc:1612] T 00000000000000000000000000000000 P c0336dff3fa14b4a829afda4fbacce67: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:25:10.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:10.014055 25836 catalog_manager.cc:1269] Loaded cluster ID: 353bedbeafb147a5bb2aac576060b9ff
16:25:10.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:10.015451 25836 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d: loading cluster ID for follower catalog manager: success
16:25:10.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:10.034897 25836 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d: acquiring CA information for follower catalog manager: success
16:25:10.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:10.041602 25836 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9b5aeb6b1dd84c09bbfc66c8d559722d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:25:10.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:10.309719 25865 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:25:10.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:10.310598 25865 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:25:10.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:10.311448 25865 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:25:10.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:10.376317 25865 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:25:10.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:10.377879 25865 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.247.1
16:25:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:10.453583 25865 tablet_server_runner.cc:78] Tablet server non-default flags:
16:25:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.247.20:37721
16:25:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-0/data
16:25:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-0/wal
16:25:10.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:10.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:10.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.247.1:0
16:25:10.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:25:10.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:10.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:10.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-0/data/info.pb
16:25:10.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.247.1
16:25:10.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:10.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.247.62:35691,127.24.247.61:34549,127.24.247.60:43119
16:25:10.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:25:10.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:10.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:25:10.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:25:10.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:10.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.247.1
16:25:10.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-0/logs
16:25:10.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:10.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:25:10.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:25:10.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:25:10.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:25:10.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:25:10.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:25:10.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:25:10.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:25:10.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:25:10.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:10.456053 25865 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:25:10.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:10.458972 25865 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:25:10.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:10.502668 25874 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:12.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:10.518069 25875 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:12.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:11.885151 25873 debug-util.cc:398] Leaking SignalData structure 0x7b080002eda0 after lost signal to thread 25865
16:25:12.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:12.577849 25865 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.062s user 0.500s sys 0.794s
16:25:12.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:12.579962 25865 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.065s user 0.501s sys 0.795s
16:25:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.593057 25865 server_base.cc:1060] running on GCE node
16:25:12.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:12.594848 25880 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:12.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.596721 25865 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:25:12.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.599512 25865 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:25:12.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.601023 25865 hybrid_clock.cc:648] HybridClock initialized: now 1773764712600786 us; error 245 us; skew 500 ppm
16:25:12.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.602123 25865 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:25:12.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.610082 25865 webserver.cc:492] Webserver started at http://127.24.247.1:36151/ using document root <none> and password file <none>
16:25:12.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.611446 25865 fs_manager.cc:362] Metadata directory not provided
16:25:12.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.611745 25865 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:25:12.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.612356 25865 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:25:12.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.618969 25865 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-0/data/instance:
16:25:12.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "08dd2bb063b9437cafe5d17f9c973e6b"
16:25:12.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:12 on dist-test-slave-jdhl"
16:25:12.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.620607 25865 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-0/wal/instance:
16:25:12.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "08dd2bb063b9437cafe5d17f9c973e6b"
16:25:12.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:12 on dist-test-slave-jdhl"
16:25:12.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.640847 25865 fs_manager.cc:696] Time spent creating directory manager: real 0.019s user 0.014s sys 0.005s
16:25:12.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.647742 25886 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:25:12.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.649293 25865 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
16:25:12.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.649726 25865 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-0/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-0/wal
16:25:12.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "08dd2bb063b9437cafe5d17f9c973e6b"
16:25:12.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:12 on dist-test-slave-jdhl"
16:25:12.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.650178 25865 fs_report.cc:389] FS layout report
16:25:12.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:25:12.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-0/wal
16:25:12.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-0/wal
16:25:12.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-0/data/data
16:25:12.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:25:12.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:25:12.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:25:12.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:25:12.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:25:12.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:25:12.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:25:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:25:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:25:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:25:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:25:12.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.715354 25865 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:25:12.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.719240 25865 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:25:12.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.720316 25865 kserver.cc:163] Server-wide thread pool size limit: 3276
16:25:12.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.735720 25865 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:25:12.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.752836 25865 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:25:12.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.763443 25865 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s user 0.001s sys 0.001s
16:25:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.764235 25865 ts_tablet_manager.cc:616] Registered 0 tablets
16:25:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:12.765007 25865 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s user 0.000s sys 0.000s
16:25:13.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.017026 25865 rpc_server.cc:307] RPC server started. Bound to: 127.24.247.1:43463
16:25:13.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.017159 25998 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.247.1:43463 every 8 connection(s)
16:25:13.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.020869 25865 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-0/data/info.pb
16:25:13.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.030366 25564 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 25865
16:25:13.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.031060 25564 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-0/wal/instance
16:25:13.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.053534 25564 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:13.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:13.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-1/wal
16:25:13.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-1/data
16:25:13.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:25:13.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:25:13.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:25:13.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:13.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-1/logs
16:25:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-1/data/info.pb
16:25:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:25:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:25:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:25:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:25:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:25:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:25:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.247.2:0
16:25:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.247.2
16:25:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.247.2
16:25:13.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:13.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.247.62:35691,127.24.247.61:34549,127.24.247.60:43119
16:25:13.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.247.20:37721
16:25:13.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:13.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:13.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:25:13.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.093526 25999 heartbeater.cc:344] Connected to a master server at 127.24.247.60:43119
16:25:13.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.094128 25999 heartbeater.cc:461] Registering TS with master...
16:25:13.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.096436 25999 heartbeater.cc:507] Master 127.24.247.60:43119 requested a full tablet report, sending...
16:25:13.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.100373 26003 heartbeater.cc:344] Connected to a master server at 127.24.247.62:35691
16:25:13.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.100521 25765 ts_manager.cc:194] Registered new tserver with Master: 08dd2bb063b9437cafe5d17f9c973e6b (127.24.247.1:43463)
16:25:13.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.100772 26003 heartbeater.cc:461] Registering TS with master...
16:25:13.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.101737 26003 heartbeater.cc:507] Master 127.24.247.62:35691 requested a full tablet report, sending...
16:25:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.105005 25609 ts_manager.cc:194] Registered new tserver with Master: 08dd2bb063b9437cafe5d17f9c973e6b (127.24.247.1:43463)
16:25:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.108601 25609 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.24.247.1:50827
16:25:13.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.117949 26000 heartbeater.cc:344] Connected to a master server at 127.24.247.61:34549
16:25:13.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.118348 26000 heartbeater.cc:461] Registering TS with master...
16:25:13.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.119071 26000 heartbeater.cc:507] Master 127.24.247.61:34549 requested a full tablet report, sending...
16:25:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.122207 25685 ts_manager.cc:194] Registered new tserver with Master: 08dd2bb063b9437cafe5d17f9c973e6b (127.24.247.1:43463)
16:25:13.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:13.434578 26009 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:25:13.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:13.435191 26009 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:25:13.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:13.435755 26009 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:25:13.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:13.472433 26009 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:25:13.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:13.473575 26009 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.247.2
16:25:13.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.515175 26009 tablet_server_runner.cc:78] Tablet server non-default flags:
16:25:13.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:13.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.247.20:37721
16:25:13.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:13.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:13.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-1/data
16:25:13.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-1/wal
16:25:13.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:13.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:13.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.247.2:0
16:25:13.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:25:13.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:13.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:13.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-1/data/info.pb
16:25:13.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.247.2
16:25:13.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.247.62:35691,127.24.247.61:34549,127.24.247.60:43119
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.247.2
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-1/logs
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:25:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.516726 26009 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:25:13.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:13.518571 26009 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:25:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:13.534145 26016 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:14.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:14.116958 26003 heartbeater.cc:499] Master 127.24.247.62:35691 was elected leader, sending a full tablet report...
16:25:15.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:14.936110 26015 debug-util.cc:398] Leaking SignalData structure 0x7b080002eda0 after lost signal to thread 26009
16:25:15.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:15.058427 26009 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.524s user 0.518s sys 0.968s
16:25:15.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:13.536898 26017 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:15.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:15.059751 26009 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.525s user 0.518s sys 0.969s
16:25:15.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:15.061470 26019 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:15.065306 26018 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1525 milliseconds
16:25:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.065456 26009 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:25:15.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.066691 26009 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:25:15.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.068921 26009 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:25:15.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.070539 26009 hybrid_clock.cc:648] HybridClock initialized: now 1773764715070504 us; error 62 us; skew 500 ppm
16:25:15.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.071558 26009 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:25:15.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.078135 26009 webserver.cc:492] Webserver started at http://127.24.247.2:42301/ using document root <none> and password file <none>
16:25:15.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.079339 26009 fs_manager.cc:362] Metadata directory not provided
16:25:15.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.079630 26009 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:25:15.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.080153 26009 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:25:15.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.085088 26009 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-1/data/instance:
16:25:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "125d5db5360d438d80def9bc54dfaeb6"
16:25:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:15 on dist-test-slave-jdhl"
16:25:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.086618 26009 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-1/wal/instance:
16:25:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "125d5db5360d438d80def9bc54dfaeb6"
16:25:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:15 on dist-test-slave-jdhl"
16:25:15.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.095556 26009 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
16:25:15.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.102061 26027 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:25:15.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.103469 26009 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
16:25:15.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.103900 26009 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-1/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-1/wal
16:25:15.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "125d5db5360d438d80def9bc54dfaeb6"
16:25:15.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:15 on dist-test-slave-jdhl"
16:25:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.104352 26009 fs_report.cc:389] FS layout report
16:25:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:25:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-1/wal
16:25:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-1/wal
16:25:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-1/data/data
16:25:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:25:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:25:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:25:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:25:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:25:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:25:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:25:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:25:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:25:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:25:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:25:15.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.201572 26009 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:25:15.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.203383 26009 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:25:15.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.203945 26009 kserver.cc:163] Server-wide thread pool size limit: 3276
16:25:15.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.206982 26009 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:25:15.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.212395 26009 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:25:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.212800 26009 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:25:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.213130 26009 ts_tablet_manager.cc:616] Registered 0 tablets
16:25:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.213400 26009 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:25:15.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.369342 26009 rpc_server.cc:307] RPC server started. Bound to: 127.24.247.2:40011
16:25:15.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.369875 26139 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.247.2:40011 every 8 connection(s)
16:25:15.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.372658 26009 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-1/data/info.pb
16:25:15.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.377321 25564 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 26009
16:25:15.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.377880 25564 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-1/wal/instance
16:25:15.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.391671 25564 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:15.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:15.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-2/wal
16:25:15.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-2/data
16:25:15.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:25:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:25:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:25:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-2/logs
16:25:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-2/data/info.pb
16:25:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:25:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:25:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:25:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:25:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:15.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:25:15.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:25:15.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.247.3:0
16:25:15.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.247.3
16:25:15.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.247.3
16:25:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.247.62:35691,127.24.247.61:34549,127.24.247.60:43119
16:25:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.247.20:37721
16:25:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:25:15.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.424993 26140 heartbeater.cc:344] Connected to a master server at 127.24.247.60:43119
16:25:15.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.425607 26140 heartbeater.cc:461] Registering TS with master...
16:25:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.427054 26140 heartbeater.cc:507] Master 127.24.247.60:43119 requested a full tablet report, sending...
16:25:15.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.429076 26141 heartbeater.cc:344] Connected to a master server at 127.24.247.61:34549
16:25:15.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.429446 26141 heartbeater.cc:461] Registering TS with master...
16:25:15.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.430447 26141 heartbeater.cc:507] Master 127.24.247.61:34549 requested a full tablet report, sending...
16:25:15.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.432978 25765 ts_manager.cc:194] Registered new tserver with Master: 125d5db5360d438d80def9bc54dfaeb6 (127.24.247.2:40011)
16:25:15.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.436384 26142 heartbeater.cc:344] Connected to a master server at 127.24.247.62:35691
16:25:15.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.436645 25685 ts_manager.cc:194] Registered new tserver with Master: 125d5db5360d438d80def9bc54dfaeb6 (127.24.247.2:40011)
16:25:15.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.436848 26142 heartbeater.cc:461] Registering TS with master...
16:25:15.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.437762 26142 heartbeater.cc:507] Master 127.24.247.62:35691 requested a full tablet report, sending...
16:25:15.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.440011 25609 ts_manager.cc:194] Registered new tserver with Master: 125d5db5360d438d80def9bc54dfaeb6 (127.24.247.2:40011)
16:25:15.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.441363 25609 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.24.247.2:59355
16:25:15.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:15.740970 26148 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:25:15.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:15.741453 26148 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:25:15.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:15.741910 26148 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:25:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:15.775046 26148 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:25:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:15.775854 26148 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.247.3
16:25:15.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.811584 26148 tablet_server_runner.cc:78] Tablet server non-default flags:
16:25:15.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:15.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.247.20:37721
16:25:15.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:15.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:15.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-2/data
16:25:15.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-2/wal
16:25:15.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:15.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:15.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.247.3:0
16:25:15.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:25:15.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-2/data/info.pb
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.247.3
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.247.62:35691,127.24.247.61:34549,127.24.247.60:43119
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.247.3
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-2/logs
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:25:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.813222 26148 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:25:15.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:15.815264 26148 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:25:15.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:15.828274 26156 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:16.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:16.445818 26142 heartbeater.cc:499] Master 127.24.247.62:35691 was elected leader, sending a full tablet report...
16:25:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:17.484582 25641 debug-util.cc:398] Leaking SignalData structure 0x7b08000aa580 after lost signal to thread 25578
16:25:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:17.485592 25641 debug-util.cc:398] Leaking SignalData structure 0x7b08000a52a0 after lost signal to thread 25644
16:25:17.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:15.828933 26157 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:17.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:17.232738 26155 debug-util.cc:398] Leaking SignalData structure 0x7b080002eda0 after lost signal to thread 26148
16:25:17.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:17.641237 26148 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.812s user 0.691s sys 1.094s
16:25:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:17.641625 26148 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.813s user 0.691s sys 1.094s
16:25:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:17.644062 26159 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:17.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:17.648834 26158 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1819 milliseconds
16:25:17.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.649371 26148 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:25:17.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.650622 26148 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:25:17.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.653314 26148 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:25:17.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.654815 26148 hybrid_clock.cc:648] HybridClock initialized: now 1773764717654724 us; error 64 us; skew 500 ppm
16:25:17.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.655714 26148 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:25:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.663153 26148 webserver.cc:492] Webserver started at http://127.24.247.3:42849/ using document root <none> and password file <none>
16:25:17.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.664271 26148 fs_manager.cc:362] Metadata directory not provided
16:25:17.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.664552 26148 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:25:17.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.665126 26148 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:25:17.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.670116 26148 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-2/data/instance:
16:25:17.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5f375f5e0bf84a2cb774db7bc893f07e"
16:25:17.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:17 on dist-test-slave-jdhl"
16:25:17.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.671810 26148 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-2/wal/instance:
16:25:17.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5f375f5e0bf84a2cb774db7bc893f07e"
16:25:17.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:17 on dist-test-slave-jdhl"
16:25:17.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.681404 26148 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.004s sys 0.005s
16:25:17.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.688766 26167 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:25:17.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.690656 26148 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
16:25:17.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.691167 26148 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-2/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-2/wal
16:25:17.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5f375f5e0bf84a2cb774db7bc893f07e"
16:25:17.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:17 on dist-test-slave-jdhl"
16:25:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.691650 26148 fs_report.cc:389] FS layout report
16:25:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:25:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-2/wal
16:25:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-2/wal
16:25:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-2/data/data
16:25:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:25:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:25:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:25:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:25:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:25:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:25:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:25:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:25:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:25:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:25:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:25:17.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.783905 26148 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:25:17.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.786183 26148 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:25:17.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.786818 26148 kserver.cc:163] Server-wide thread pool size limit: 3276
16:25:17.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.790454 26148 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:25:17.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.796993 26148 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:25:17.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.797442 26148 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:25:17.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.797787 26148 ts_tablet_manager.cc:616] Registered 0 tablets
16:25:17.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.798038 26148 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:25:17.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.970851 26148 rpc_server.cc:307] RPC server started. Bound to: 127.24.247.3:35949
16:25:17.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.970983 26279 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.247.3:35949 every 8 connection(s)
16:25:17.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.975016 26148 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-2/data/info.pb
16:25:17.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.975653 25564 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 26148
16:25:17.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:17.976070 25564 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11627491050288468197/ts-2/wal/instance
16:25:18.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:18.046442 26281 heartbeater.cc:344] Connected to a master server at 127.24.247.61:34549
16:25:18.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:18.047051 26281 heartbeater.cc:461] Registering TS with master...
16:25:18.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:18.048537 26281 heartbeater.cc:507] Master 127.24.247.61:34549 requested a full tablet report, sending...
16:25:18.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:18.049407 26280 heartbeater.cc:344] Connected to a master server at 127.24.247.60:43119
16:25:18.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:18.049752 26280 heartbeater.cc:461] Registering TS with master...
16:25:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:18.050741 26280 heartbeater.cc:507] Master 127.24.247.60:43119 requested a full tablet report, sending...
16:25:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:18.051946 25685 ts_manager.cc:194] Registered new tserver with Master: 5f375f5e0bf84a2cb774db7bc893f07e (127.24.247.3:35949)
16:25:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:18.053481 25765 ts_manager.cc:194] Registered new tserver with Master: 5f375f5e0bf84a2cb774db7bc893f07e (127.24.247.3:35949)
16:25:18.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:18.057196 26283 heartbeater.cc:344] Connected to a master server at 127.24.247.62:35691
16:25:18.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:18.057624 26283 heartbeater.cc:461] Registering TS with master...
16:25:18.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:18.058633 26283 heartbeater.cc:507] Master 127.24.247.62:35691 requested a full tablet report, sending...
16:25:18.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:18.061376 25609 ts_manager.cc:194] Registered new tserver with Master: 5f375f5e0bf84a2cb774db7bc893f07e (127.24.247.3:35949)
16:25:18.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:18.063269 25609 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.24.247.3:43881
16:25:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:18.071943 25564 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
16:25:18.090 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:25:18.092 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
16:25:18.106 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.24.247.62"
port: 35691
}
bound_http_address {
host: "127.24.247.62"
port: 33409
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.24.247.61"
port: 34549
}
bound_http_address {
host: "127.24.247.61"
port: 33079
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.24.247.60"
port: 43119
}
bound_http_address {
host: "127.24.247.60"
port: 41809
}
}
}
16:25:18.111 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:25:18.116 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.24.247.1"
port: 43463
}
bound_http_address {
host: "127.24.247.1"
port: 36151
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.24.247.2"
port: 40011
}
bound_http_address {
host: "127.24.247.2"
port: 42301
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.24.247.3"
port: 35949
}
bound_http_address {
host: "127.24.247.3"
port: 42849
}
}
}
16:25:18.116 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
16:25:18.343 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
16:25:18.435 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
16:25:18.577 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskMUvRhB/test-tmp
16:25:18.578 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
16:25:18.578 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskMUvRhB/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
16:25:18.583 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
16:25:18.997 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:25:18.997 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:25:19.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:19.003242 26292 mini_oidc.cc:150] Starting JWKS server
16:25:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:19.007576 26292 webserver.cc:492] Webserver started at http://127.0.0.1:33477/ using document root <none> and password file <none>
16:25:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:19.008244 26292 mini_oidc.cc:186] Starting OIDC Discovery server
16:25:19.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:19.010492 26292 webserver.cc:492] Webserver started at http://127.0.0.1:44939/ using document root <none> and password file <none>
16:25:19.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:19.067281 26283 heartbeater.cc:499] Master 127.24.247.62:35691 was elected leader, sending a full tablet report...
16:25:19.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
16:25:19.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/principal' for realm 'KRBTEST.COM',
16:25:19.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
16:25:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:19 dist-test-slave-jdhl krb5kdc[26304](info): setting up network...
16:25:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
16:25:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:19 dist-test-slave-jdhl krb5kdc[26304](info): set up 2 sockets
16:25:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:19 dist-test-slave-jdhl krb5kdc[26304](info): commencing operation
16:25:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
16:25:21.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:21.284703 26292 mini_kdc.cc:121] Time spent starting KDC: real 2.221s user 0.004s sys 0.021s
16:25:21.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
16:25:21.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:25:21.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
16:25:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:21.441092 26292 mini_kdc.cc:253] Time spent creating user principal test-admin: real 0.156s user 0.001s sys 0.004s
16:25:21.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
16:25:21.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:25:21.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
16:25:21.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:21.691399 26292 mini_kdc.cc:253] Time spent creating user principal test-user: real 0.250s user 0.001s sys 0.005s
16:25:21.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
16:25:22.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:25:22.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
16:25:22.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:22.002633 26292 mini_kdc.cc:253] Time spent creating user principal joe-interloper: real 0.311s user 0.001s sys 0.004s
16:25:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:25:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/test-user.keytab.
16:25:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/test-user.keytab.
16:25:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:22.988126 26292 mini_kdc.cc:290] Time spent creating keytab for test-user: real 0.985s user 0.002s sys 0.003s
16:25:23.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:23 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764723, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:23.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:25:23.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-17T16:25:23Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
16:25:23.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-17T16:25:23Z Disabled control of system clock
16:25:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.173.62@KRBTEST.COM; defaulting to no policy
16:25:23.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:23.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.173.62@KRBTEST.COM" created.
16:25:23.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:23.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.173.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:23.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.173.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:23.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:23.369035 26292 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.25.173.62: real 0.319s user 0.000s sys 0.012s
16:25:23.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.25.173.62@KRBTEST.COM; defaulting to no policy
16:25:23.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:23.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.25.173.62@KRBTEST.COM" created.
16:25:23.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:23.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.25.173.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:23.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.25.173.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:23.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:23.413745 26292 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:23.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:23.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/wal
16:25:23.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data
16:25:23.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:25:23.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:25:23.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:25:23.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:23.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:23.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:23.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/logs
16:25:23.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data/info.pb
16:25:23.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:23.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.62:36591
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.62
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.62
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33477/jwks.json
16:25:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:25:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:25:23.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:23.743227 26326 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:25:23.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:23.743913 26326 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:25:23.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:23.744998 26326 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:25:23.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:23.777933 26326 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:25:23.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:23.778249 26326 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:25:23.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:23.778681 26326 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:25:23.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:23.779039 26326 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:25:23.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:23.779356 26326 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:25:23.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:23.779646 26326 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:25:23.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:23.779871 26326 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33477/jwks.json
16:25:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:23.819042 26326 master_runner.cc:386] Master server non-default flags:
16:25:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data
16:25:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/wal
16:25:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:25:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.62
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.62:36591
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33477/jwks.json
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data/info.pb
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.62
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:25:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:25:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/logs
16:25:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:25:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:25:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:25:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:25:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:25:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:25:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:25:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:25:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:25:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:23.820832 26326 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:25:23.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:23.822149 26326 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:25:23.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:23.835412 26332 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:23.836009 26333 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:25.238296 26331 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 26326
16:25:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:25.684532 26331 kernel_stack_watchdog.cc:198] Thread 26326 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
16:25:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:25:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:25:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:25:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:25:25.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:25.690661 26326 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.856s user 0.636s sys 1.182s
16:25:25.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:25.691118 26326 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.857s user 0.637s sys 1.182s
16:25:25.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:25.692382 26334 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1857 milliseconds
16:25:25.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.693799 26326 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:25:25.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:25.693763 26335 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:25.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.695966 26326 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:25:25.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.698637 26326 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:25:25.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.700434 26326 hybrid_clock.cc:648] HybridClock initialized: now 1773764725700355 us; error 84 us; skew 500 ppm
16:25:25.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.701563 26326 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:25:25.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:25 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764725, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.709182 26326 init.cc:377] Logged in from keytab as kudu/127.25.173.62@KRBTEST.COM (short username kudu)
16:25:25.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.714635 26326 webserver.cc:492] Webserver started at http://127.25.173.62:41617/ using document root <none> and password file <none>
16:25:25.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.715977 26326 fs_manager.cc:362] Metadata directory not provided
16:25:25.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.716243 26326 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:25:25.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.716663 26326 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:25:25.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.722546 26326 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data/instance:
16:25:25.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9bddceeba6654533b175280b4491784f"
16:25:25.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:25 on dist-test-slave-jdhl"
16:25:25.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.724334 26326 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/wal/instance:
16:25:25.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9bddceeba6654533b175280b4491784f"
16:25:25.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:25 on dist-test-slave-jdhl"
16:25:25.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.737399 26326 fs_manager.cc:696] Time spent creating directory manager: real 0.012s user 0.010s sys 0.000s
16:25:25.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.743793 26345 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:25:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.745326 26326 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
16:25:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.745740 26326 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/wal
16:25:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9bddceeba6654533b175280b4491784f"
16:25:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:25 on dist-test-slave-jdhl"
16:25:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.746065 26326 fs_report.cc:389] FS layout report
16:25:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:25:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/wal
16:25:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/wal
16:25:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data/data
16:25:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:25:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:25:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:25:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:25:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:25:25.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:25:25.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:25:25.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:25:25.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:25:25.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:25:25.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:25:25.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.811911 26326 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:25:25.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.823444 26326 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:25:25.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.824002 26326 kserver.cc:163] Server-wide thread pool size limit: 3276
16:25:25.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.902395 26326 rpc_server.cc:307] RPC server started. Bound to: 127.25.173.62:36591
16:25:25.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.902446 26396 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.173.62:36591 every 8 connection(s)
16:25:25.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.905839 26326 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data/info.pb
16:25:25.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.913249 26292 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 26326
16:25:25.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.913767 26292 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/wal/instance
16:25:25.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.913954 26397 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:25:25.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.925372 26397 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } has no permanent_uuid. Determining permanent_uuid...
16:25:25.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.173.61@KRBTEST.COM; defaulting to no policy
16:25:25.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:25.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.173.61@KRBTEST.COM" created.
16:25:25.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:25 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764725, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.62@KRBTEST.COM for kudu/127.25.173.62@KRBTEST.COM
16:25:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.173.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.173.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:25.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:25.973271 26397 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } has no permanent_uuid. Determining permanent_uuid...
16:25:25.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:25.976612 26347 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.173.61:39669: connect: Connection refused (error 111)
16:25:25.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.25.173.61@KRBTEST.COM; defaulting to no policy
16:25:25.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:25.981673 26397 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.61:39669: Network error: Client connection negotiation failed: client connection to 127.25.173.61:39669: connect: Connection refused (error 111)
16:25:25.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:25.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.25.173.61@KRBTEST.COM" created.
16:25:26.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:26.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.25.173.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:26.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.25.173.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:26.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:26.008407 26292 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:26.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/wal
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/logs
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data/info.pb
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:25:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:25:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.61:39669
16:25:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.61
16:25:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.61
16:25:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:25:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:25:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:25:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:26.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:26.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:26.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:26.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33477/jwks.json
16:25:26.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:25:26.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:25:26.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:26.031184 26397 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } attempt: 1
16:25:26.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:26.035687 26397 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.61:39669: Network error: Client connection negotiation failed: client connection to 127.25.173.61:39669: connect: Connection refused (error 111)
16:25:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:26.104149 26397 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } attempt: 2
16:25:26.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:26.110987 26397 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.61:39669: Network error: Client connection negotiation failed: client connection to 127.25.173.61:39669: connect: Connection refused (error 111)
16:25:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:26.202525 26397 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } attempt: 3
16:25:26.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:26.206876 26397 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.61:39669: Network error: Client connection negotiation failed: client connection to 127.25.173.61:39669: connect: Connection refused (error 111)
16:25:26.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:26.350279 26397 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } attempt: 4
16:25:26.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:26.354282 26406 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:25:26.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:26.355295 26406 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:25:26.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:26.355813 26397 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.61:39669: Network error: Client connection negotiation failed: client connection to 127.25.173.61:39669: connect: Connection refused (error 111)
16:25:26.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:26.356336 26406 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:25:26.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:26.389381 26406 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:25:26.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:26.389700 26406 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:25:26.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:26.390139 26406 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:25:26.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:26.390483 26406 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:25:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:26.390820 26406 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:25:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:26.391137 26406 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:25:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:26.391386 26406 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33477/jwks.json
16:25:26.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:26.429092 26406 master_runner.cc:386] Master server non-default flags:
16:25:26.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:26.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:26.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:26.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:26.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data
16:25:26.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/wal
16:25:26.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:25:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.61
16:25:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.61:39669
16:25:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:25:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33477/jwks.json
16:25:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:25:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:25:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data/info.pb
16:25:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.61
16:25:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:25:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:25:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:25:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/logs
16:25:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:25:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:25:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:25:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:25:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:25:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:25:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:25:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:25:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:25:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:26.430877 26406 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:25:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:26.432303 26406 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:25:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:26.446863 26412 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:26.655289 26397 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } attempt: 5
16:25:26.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:26.663563 26397 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.61:39669: Network error: Client connection negotiation failed: client connection to 127.25.173.61:39669: connect: Connection refused (error 111)
16:25:27.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:27.211149 26397 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } attempt: 6
16:25:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:27.237521 26397 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.61:39669: Network error: Client connection negotiation failed: client connection to 127.25.173.61:39669: connect: Connection refused (error 111)
16:25:28.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.298120 26397 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } attempt: 7
16:25:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:28.322273 26397 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.61:39669: Network error: Client connection negotiation failed: client connection to 127.25.173.61:39669: connect: Connection refused (error 111)
16:25:28.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:27.849002 26411 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 26406
16:25:28.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:28.326807 26406 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.880s user 0.666s sys 1.137s
16:25:28.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:26.447593 26413 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:28.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:28.327960 26406 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.880s user 0.666s sys 1.137s
16:25:28.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:28.328727 26414 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1880 milliseconds
16:25:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:28.330391 26415 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:28.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.330694 26406 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:25:28.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.336680 26406 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:25:28.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.339408 26406 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:25:28.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.340909 26406 hybrid_clock.cc:648] HybridClock initialized: now 1773764728340831 us; error 90 us; skew 500 ppm
16:25:28.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.341848 26406 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:25:28.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:28 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764728, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.349205 26406 init.cc:377] Logged in from keytab as kudu/127.25.173.61@KRBTEST.COM (short username kudu)
16:25:28.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.356567 26406 webserver.cc:492] Webserver started at http://127.25.173.61:46803/ using document root <none> and password file <none>
16:25:28.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.357726 26406 fs_manager.cc:362] Metadata directory not provided
16:25:28.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.358032 26406 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:25:28.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.358667 26406 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:25:28.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.363572 26406 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data/instance:
16:25:28.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9555719ea49a4994a3973eb4452a45df"
16:25:28.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:28 on dist-test-slave-jdhl"
16:25:28.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.365466 26406 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/wal/instance:
16:25:28.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9555719ea49a4994a3973eb4452a45df"
16:25:28.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:28 on dist-test-slave-jdhl"
16:25:28.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.377473 26406 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.004s sys 0.008s
16:25:28.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.385751 26429 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:25:28.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.387326 26406 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.002s
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.387656 26406 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/wal
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9555719ea49a4994a3973eb4452a45df"
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:28 on dist-test-slave-jdhl"
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.387979 26406 fs_report.cc:389] FS layout report
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/wal
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/wal
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data/data
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:25:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:25:28.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.450100 26406 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:25:28.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.457090 26406 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:25:28.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.457687 26406 kserver.cc:163] Server-wide thread pool size limit: 3276
16:25:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.565198 26406 rpc_server.cc:307] RPC server started. Bound to: 127.25.173.61:39669
16:25:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.565352 26480 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.173.61:39669 every 8 connection(s)
16:25:28.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.568222 26406 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data/info.pb
16:25:28.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.575416 26292 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 26406
16:25:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.575878 26292 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/wal/instance
16:25:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.576273 26481 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:25:28.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.589720 26481 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } has no permanent_uuid. Determining permanent_uuid...
16:25:28.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.173.60@KRBTEST.COM; defaulting to no policy
16:25:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.173.60@KRBTEST.COM" created.
16:25:28.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:28 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764728, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.61@KRBTEST.COM for kudu/127.25.173.62@KRBTEST.COM
16:25:28.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:28.776031 26481 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } has no permanent_uuid. Determining permanent_uuid...
16:25:28.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:28 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764728, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.61@KRBTEST.COM for kudu/127.25.173.61@KRBTEST.COM
16:25:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.173.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.173.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:29.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.402737 26292 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.25.173.60: real 0.825s user 0.003s sys 0.010s
16:25:29.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.404893 26487 sasl_common.cc:323] Time spent calling sasl_server_start(): real 0.433s user 0.002s sys 0.002s
16:25:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:29.412675 26481 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } has no permanent_uuid. Determining permanent_uuid...
16:25:29.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.415098 26431 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.173.60:36837: connect: Connection refused (error 111)
16:25:29.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.25.173.60@KRBTEST.COM; defaulting to no policy
16:25:29.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.419318 26481 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.60:36837: Network error: Client connection negotiation failed: client connection to 127.25.173.60:36837: connect: Connection refused (error 111)
16:25:29.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:29.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.25.173.60@KRBTEST.COM" created.
16:25:29.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:29.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.25.173.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:29.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.25.173.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:29.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:29.448081 26292 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:29.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:29.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/wal
16:25:29.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data
16:25:29.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:25:29.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:25:29.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:25:29.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:29.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:29.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/logs
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data/info.pb
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.60:36837
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.60
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.60
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:25:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33477/jwks.json
16:25:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:25:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:25:29.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:29.468814 26481 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } attempt: 1
16:25:29.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.472913 26481 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.60:36837: Network error: Client connection negotiation failed: client connection to 127.25.173.60:36837: connect: Connection refused (error 111)
16:25:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:29.541379 26481 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } attempt: 2
16:25:29.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.545831 26481 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.60:36837: Network error: Client connection negotiation failed: client connection to 127.25.173.60:36837: connect: Connection refused (error 111)
16:25:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:29.637364 26481 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } attempt: 3
16:25:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.642279 26481 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.60:36837: Network error: Client connection negotiation failed: client connection to 127.25.173.60:36837: connect: Connection refused (error 111)
16:25:29.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:29.785699 26481 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } attempt: 4
16:25:29.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.790405 26481 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.60:36837: Network error: Client connection negotiation failed: client connection to 127.25.173.60:36837: connect: Connection refused (error 111)
16:25:29.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.798185 26492 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:25:29.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.798727 26492 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:25:29.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.799420 26492 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:25:29.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.840045 26492 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:25:29.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.840404 26492 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:25:29.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.840943 26492 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:25:29.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.841493 26492 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:25:29.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.841907 26492 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:25:29.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.842496 26492 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:25:29.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.842916 26492 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33477/jwks.json
16:25:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:29.900712 26492 master_runner.cc:386] Master server non-default flags:
16:25:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data
16:25:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/wal
16:25:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:25:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.60
16:25:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.60:36837
16:25:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33477/jwks.json
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data/info.pb
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.60
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/logs
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:25:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:25:29.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:25:29.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:25:29.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:25:29.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:25:29.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:29.902454 26492 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:25:29.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:29.903888 26492 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:25:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.916118 26498 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:30.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:30.089821 26481 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } attempt: 5
16:25:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:30.095496 26481 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.60:36837: Network error: Client connection negotiation failed: client connection to 127.25.173.60:36837: connect: Connection refused (error 111)
16:25:30.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:30.412734 26397 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } attempt: 8
16:25:30.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:30 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764725, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.62@KRBTEST.COM for kudu/127.25.173.61@KRBTEST.COM
16:25:30.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:30.436647 26397 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } has no permanent_uuid. Determining permanent_uuid...
16:25:30.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:30.444080 26397 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.60:36837: Network error: Client connection negotiation failed: client connection to 127.25.173.60:36837: connect: Connection refused (error 111)
16:25:30.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:30.509805 26397 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } attempt: 1
16:25:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:30.514353 26397 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.60:36837: Network error: Client connection negotiation failed: client connection to 127.25.173.60:36837: connect: Connection refused (error 111)
16:25:30.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:30.568027 26397 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } attempt: 2
16:25:30.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:30.575264 26397 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.60:36837: Network error: Client connection negotiation failed: client connection to 127.25.173.60:36837: connect: Connection refused (error 111)
16:25:30.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:30.643056 26481 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } attempt: 6
16:25:30.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:30.650979 26481 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.60:36837: Network error: Client connection negotiation failed: client connection to 127.25.173.60:36837: connect: Connection refused (error 111)
16:25:30.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:30.651791 26397 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } attempt: 3
16:25:30.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:30.657614 26397 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.60:36837: Network error: Client connection negotiation failed: client connection to 127.25.173.60:36837: connect: Connection refused (error 111)
16:25:30.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:30.813016 26397 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } attempt: 4
16:25:30.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:30.817413 26397 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.60:36837: Network error: Client connection negotiation failed: client connection to 127.25.173.60:36837: connect: Connection refused (error 111)
16:25:31.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.114259 26397 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } attempt: 5
16:25:31.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:31.116466 26347 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.173.60:36837: connect: Connection refused (error 111) [suppressed 12 similar messages]
16:25:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:31.118724 26397 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.173.60:36837: Network error: Client connection negotiation failed: client connection to 127.25.173.60:36837: connect: Connection refused (error 111)
16:25:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.923123 26501 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:29.924458 26492 server_base.cc:1060] running on GCE node
16:25:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:29.921084 26499 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:31.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.204802 26492 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:25:31.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.208976 26492 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:25:31.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.210850 26492 hybrid_clock.cc:648] HybridClock initialized: now 1773764731210709 us; error 50 us; skew 500 ppm
16:25:31.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.211881 26492 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:25:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:31 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764731, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.218762 26492 init.cc:377] Logged in from keytab as kudu/127.25.173.60@KRBTEST.COM (short username kudu)
16:25:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.225364 26492 webserver.cc:492] Webserver started at http://127.25.173.60:41079/ using document root <none> and password file <none>
16:25:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.226377 26492 fs_manager.cc:362] Metadata directory not provided
16:25:31.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.226652 26492 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:25:31.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.227214 26492 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:25:31.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.232961 26492 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data/instance:
16:25:31.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7724600eba9e4ebea1a0a1f3720ac88c"
16:25:31.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:31 on dist-test-slave-jdhl"
16:25:31.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.234177 26492 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/wal/instance:
16:25:31.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7724600eba9e4ebea1a0a1f3720ac88c"
16:25:31.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:31 on dist-test-slave-jdhl"
16:25:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.242683 26492 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.010s sys 0.001s
16:25:31.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.249464 26516 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:25:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.251082 26492 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.004s
16:25:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.251470 26492 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/wal
16:25:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7724600eba9e4ebea1a0a1f3720ac88c"
16:25:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:31 on dist-test-slave-jdhl"
16:25:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.251796 26492 fs_report.cc:389] FS layout report
16:25:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:25:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/wal
16:25:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/wal
16:25:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data/data
16:25:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:25:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:25:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:25:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:25:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:25:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:25:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:25:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:25:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:25:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:25:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:25:31.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.305116 26492 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:25:31.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.316679 26492 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:25:31.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.317130 26492 kserver.cc:163] Server-wide thread pool size limit: 3276
16:25:31.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.420562 26492 rpc_server.cc:307] RPC server started. Bound to: 127.25.173.60:36837
16:25:31.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.420635 26567 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.173.60:36837 every 8 connection(s)
16:25:31.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.423450 26492 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data/info.pb
16:25:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.425627 26292 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 26492
16:25:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.426183 26292 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/wal/instance
16:25:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.432665 26568 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:25:31.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.453526 26568 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } has no permanent_uuid. Determining permanent_uuid...
16:25:31.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:31 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764723, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.173.62@KRBTEST.COM
16:25:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:31 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764731, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.60@KRBTEST.COM for kudu/127.25.173.62@KRBTEST.COM
16:25:31.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:31 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764723, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.173.61@KRBTEST.COM
16:25:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.506162 26568 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } has no permanent_uuid. Determining permanent_uuid...
16:25:31.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:31 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764731, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.60@KRBTEST.COM for kudu/127.25.173.61@KRBTEST.COM
16:25:31.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:31 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764723, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.173.60@KRBTEST.COM
16:25:31.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.550136 26568 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } has no permanent_uuid. Determining permanent_uuid...
16:25:31.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:31 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764731, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.60@KRBTEST.COM for kudu/127.25.173.60@KRBTEST.COM
16:25:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.591886 26568 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: Bootstrap starting.
16:25:31.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.601224 26568 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: Neither blocks nor log segments found. Creating new log.
16:25:31.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.604873 26568 log.cc:826] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: Log is configured to *not* fsync() on all Append() calls
16:25:31.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.615342 26568 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: No bootstrap required, opened a new log
16:25:31.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.640261 26397 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } attempt: 6
16:25:31.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.653375 26568 raft_consensus.cc:359] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:31.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.654515 26568 raft_consensus.cc:385] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:25:31.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.655094 26568 raft_consensus.cc:740] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7724600eba9e4ebea1a0a1f3720ac88c, State: Initialized, Role: FOLLOWER
16:25:31.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.656266 26568 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:31.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:31 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764725, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.62@KRBTEST.COM for kudu/127.25.173.60@KRBTEST.COM
16:25:31.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.664492 26579 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:31.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.666152 26579 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: This master's current role is: FOLLOWER
16:25:31.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.669965 26568 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: configured and running, proceeding with master startup.
16:25:31.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.710966 26397 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: Bootstrap starting.
16:25:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.711675 26481 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } attempt: 7
16:25:31.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.752350 26397 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: Neither blocks nor log segments found. Creating new log.
16:25:31.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.759932 26397 log.cc:826] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: Log is configured to *not* fsync() on all Append() calls
16:25:31.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:31 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764728, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.61@KRBTEST.COM for kudu/127.25.173.60@KRBTEST.COM
16:25:31.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.790256 26397 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: No bootstrap required, opened a new log
16:25:31.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.801476 26481 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: Bootstrap starting.
16:25:31.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.813373 26481 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: Neither blocks nor log segments found. Creating new log.
16:25:31.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.816413 26481 log.cc:826] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: Log is configured to *not* fsync() on all Append() calls
16:25:31.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:31.824914 26522 tablet.cc:2379] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:25:31.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.832476 26481 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: No bootstrap required, opened a new log
16:25:31.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.839931 26397 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:31.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.841281 26397 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:25:31.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.841701 26397 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9bddceeba6654533b175280b4491784f, State: Initialized, Role: FOLLOWER
16:25:31.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.843225 26397 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:31.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:31.853737 26593 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:25:31.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:31.856484 26593 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:25:31.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.867040 26596 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.867841 26596 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [sys.catalog]: This master's current role is: FOLLOWER
16:25:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.869597 26579 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:25:31.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.870288 26579 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:31.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.873874 26397 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [sys.catalog]: configured and running, proceeding with master startup.
16:25:31.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.881384 26372 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9bddceeba6654533b175280b4491784f" is_pre_election: true
16:25:31.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.883082 26372 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7724600eba9e4ebea1a0a1f3720ac88c in term 0.
16:25:31.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.883381 26481 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:31.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.884377 26481 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:25:31.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.885394 26481 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9555719ea49a4994a3973eb4452a45df, State: Initialized, Role: FOLLOWER
16:25:31.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.885668 26520 leader_election.cc:304] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7724600eba9e4ebea1a0a1f3720ac88c, 9bddceeba6654533b175280b4491784f; no voters:
16:25:31.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.888362 26481 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:31.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.890697 26579 leader_election.cc:290] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9bddceeba6654533b175280b4491784f (127.25.173.62:36591), 9555719ea49a4994a3973eb4452a45df (127.25.173.61:39669)
16:25:31.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.891587 26579 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 0 FOLLOWER]: Leader pre-election won for term 1
16:25:31.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.891947 26579 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:25:31.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.892452 26579 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 0 FOLLOWER]: Advancing to term 1
16:25:31.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.916042 26481 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: configured and running, proceeding with master startup.
16:25:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.918803 26456 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9555719ea49a4994a3973eb4452a45df" is_pre_election: true
16:25:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.919967 26456 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7724600eba9e4ebea1a0a1f3720ac88c in term 0.
16:25:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.932417 26601 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.935720 26601 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: This master's current role is: FOLLOWER
16:25:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.935724 26579 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:31.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.941859 26456 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9555719ea49a4994a3973eb4452a45df"
16:25:31.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.942658 26456 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 0 FOLLOWER]: Advancing to term 1
16:25:31.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.941649 26579 leader_election.cc:290] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 1 election: Requested vote from peers 9bddceeba6654533b175280b4491784f (127.25.173.62:36591), 9555719ea49a4994a3973eb4452a45df (127.25.173.61:39669)
16:25:31.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.951725 26372 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9bddceeba6654533b175280b4491784f"
16:25:31.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.952327 26372 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [term 0 FOLLOWER]: Advancing to term 1
16:25:31.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.952955 26456 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7724600eba9e4ebea1a0a1f3720ac88c in term 1.
16:25:31.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.826119 26492 master_runner.cc:186] Error getting master registration for 127.25.173.62:36591: OK, instance_id { permanent_uuid: "9bddceeba6654533b175280b4491784f" instance_seqno: 1773764725878579 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:25:31.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.957026 26518 leader_election.cc:304] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7724600eba9e4ebea1a0a1f3720ac88c, 9555719ea49a4994a3973eb4452a45df; no voters:
16:25:31.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.958647 26579 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 1 FOLLOWER]: Leader election won for term 1
16:25:31.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.959923 26492 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:25:31.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.961004 26579 raft_consensus.cc:697] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 1 LEADER]: Becoming Leader. State: Replica: 7724600eba9e4ebea1a0a1f3720ac88c, State: Running, Role: LEADER
16:25:31.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.962366 26579 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:31.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:31.969995 26435 tablet.cc:2379] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:25:31.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.981389 26372 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7724600eba9e4ebea1a0a1f3720ac88c in term 1.
16:25:31.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.994421 26599 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7724600eba9e4ebea1a0a1f3720ac88c. Latest consensus state: current_term: 1 leader_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:31.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:31.997066 26599 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: This master's current role is: LEADER
16:25:32.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.000195 26613 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
16:25:32.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.030135 26613 catalog_manager.cc:1494] Initializing Kudu cluster ID...
16:25:32.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:32.052371 26618 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:25:32.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:32.054481 26618 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:25:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.076555 26456 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 1 FOLLOWER]: Refusing update from remote peer 7724600eba9e4ebea1a0a1f3720ac88c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:25:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.078740 26599 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [LEADER]: Connected to new peer: Peer: permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
16:25:32.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:32.086828 26351 tablet.cc:2379] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:25:32.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.093850 26372 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [term 1 FOLLOWER]: Refusing update from remote peer 7724600eba9e4ebea1a0a1f3720ac88c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:25:32.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.097684 26599 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [LEADER]: Connected to new peer: Peer: permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:25:32.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.040293 26326 master_runner.cc:186] Error getting master registration for 127.25.173.61:39669: OK, instance_id { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" instance_seqno: 1773764728534392 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:25:32.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.103138 26326 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:25:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.152083 26601 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7724600eba9e4ebea1a0a1f3720ac88c. Latest consensus state: current_term: 1 leader_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.155143 26601 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: This master's current role is: FOLLOWER
16:25:32.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.164144 26621 mvcc.cc:204] Tried to move back new op lower bound from 7265340342560636928 to 7265340342139424768. Current Snapshot: MvccSnapshot[applied={T|T < 7265340342560636928}]
16:25:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.195580 26596 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7724600eba9e4ebea1a0a1f3720ac88c. Latest consensus state: current_term: 1 leader_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.198073 26596 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [sys.catalog]: This master's current role is: FOLLOWER
16:25:32.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.207363 26630 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:32.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.208563 26630 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [sys.catalog]: This master's current role is: FOLLOWER
16:25:32.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.209841 26599 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:32.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.210877 26599 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: This master's current role is: LEADER
16:25:32.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.211611 26599 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:32.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.212286 26599 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: This master's current role is: LEADER
16:25:32.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.215775 26613 catalog_manager.cc:1357] Generated new cluster ID: 723802fca27f462db7efb81f8e18a050
16:25:32.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.216096 26613 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
16:25:32.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.217994 26601 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:32.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.218714 26601 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: This master's current role is: FOLLOWER
16:25:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.250062 26628 catalog_manager.cc:1269] Loaded cluster ID: 723802fca27f462db7efb81f8e18a050
16:25:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.250735 26628 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: loading cluster ID for follower catalog manager: success
16:25:32.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:32.255673 26628 catalog_manager.cc:1586] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: acquiring CA information for follower catalog manager: Not found: root CA entry not found
16:25:32.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:32.259215 26628 catalog_manager.cc:883] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry
16:25:32.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.289122 26613 catalog_manager.cc:1380] Generated new certificate authority record
16:25:32.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.329859 26613 catalog_manager.cc:1514] Loading token signing keys...
16:25:32.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.372076 26613 catalog_manager.cc:6044] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: Generated new TSK 0
16:25:32.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.373311 26613 catalog_manager.cc:1524] Initializing in-progress tserver states...
16:25:32.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.173.1@KRBTEST.COM; defaulting to no policy
16:25:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.173.1@KRBTEST.COM" created.
16:25:32.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:32.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.173.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:32.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.173.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:32.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:32.561398 26292 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.25.173.1: real 0.125s user 0.005s sys 0.013s
16:25:32.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.25.173.1@KRBTEST.COM; defaulting to no policy
16:25:32.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:32.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.25.173.1@KRBTEST.COM" created.
16:25:32.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:32.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.25.173.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:32.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.25.173.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:32.640565 26292 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-0/wal
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-0/data
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-0/logs
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-0/data/info.pb
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:25:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:25:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.1:0
16:25:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.173.1
16:25:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.1
16:25:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.1
16:25:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:25:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:25:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:25:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:25:33.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:33.065238 26618 catalog_manager.cc:1269] Loaded cluster ID: 723802fca27f462db7efb81f8e18a050
16:25:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:33.066159 26618 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: loading cluster ID for follower catalog manager: success
16:25:33.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:33.074615 26618 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: acquiring CA information for follower catalog manager: success
16:25:33.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:33.092753 26618 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:25:33.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:33.285250 26628 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: acquiring CA information for follower catalog manager: success
16:25:33.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:33.301445 26628 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:25:33.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:33.311286 26639 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:25:33.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:33.311944 26639 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:25:33.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:33.313022 26639 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:25:33.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:33.314211 26477 debug-util.cc:398] Leaking SignalData structure 0x7b080007a3a0 after lost signal to thread 26407
16:25:33.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:33.314834 26477 debug-util.cc:398] Leaking SignalData structure 0x7b0800075fc0 after lost signal to thread 26480
16:25:33.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:33.429674 26639 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:25:33.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:33.445101 26639 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.25.173.1
16:25:33.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:33.637005 26639 tablet_server_runner.cc:78] Tablet server non-default flags:
16:25:33.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:33.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:33.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:33.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:33.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-0/data
16:25:33.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-0/wal
16:25:33.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.1
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.1:0
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-0/data/info.pb
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.1
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.173.1
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-0/logs
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:25:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:25:33.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:25:33.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:25:33.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:25:33.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:25:33.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:25:33.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:33.640717 26639 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:25:33.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:33.642473 26639 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:25:33.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:33.715628 26647 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:35.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:35 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764735, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:35.717365 26340 init.cc:295] Successfully reacquired a new kerberos TGT
16:25:36.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:33.719324 26648 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:33.739598 26651 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:36.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:36.392405 26649 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2676 milliseconds
16:25:36.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.392535 26639 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:25:36.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.396857 26639 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:25:36.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.410248 26639 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:25:36.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.421489 26639 hybrid_clock.cc:648] HybridClock initialized: now 1773764736421412 us; error 73 us; skew 500 ppm
16:25:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.423089 26639 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:25:36.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:36 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764736, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:36.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.445714 26639 init.cc:377] Logged in from keytab as kudu/127.25.173.1@KRBTEST.COM (short username kudu)
16:25:36.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.470263 26639 webserver.cc:492] Webserver started at http://127.25.173.1:34185/ using document root <none> and password file <none>
16:25:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.471753 26639 fs_manager.cc:362] Metadata directory not provided
16:25:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.472096 26639 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:25:36.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.472666 26639 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:25:36.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.480301 26639 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-0/data/instance:
16:25:36.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3be9c2934e634e9db2674aa1fcd019af"
16:25:36.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:36 on dist-test-slave-jdhl"
16:25:36.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.482049 26639 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-0/wal/instance:
16:25:36.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3be9c2934e634e9db2674aa1fcd019af"
16:25:36.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:36 on dist-test-slave-jdhl"
16:25:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.501266 26639 fs_manager.cc:696] Time spent creating directory manager: real 0.018s user 0.006s sys 0.008s
16:25:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.518884 26664 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:25:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.520346 26639 fs_manager.cc:730] Time spent opening block manager: real 0.012s user 0.005s sys 0.004s
16:25:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.520866 26639 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-0/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-0/wal
16:25:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3be9c2934e634e9db2674aa1fcd019af"
16:25:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:36 on dist-test-slave-jdhl"
16:25:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.521401 26639 fs_report.cc:389] FS layout report
16:25:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:25:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-0/wal
16:25:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-0/wal
16:25:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-0/data/data
16:25:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:25:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:25:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:25:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:25:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:25:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:25:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:25:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:25:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:25:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:25:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:25:36.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.729532 26639 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:25:36.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.751346 26639 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:25:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.751933 26639 kserver.cc:163] Server-wide thread pool size limit: 3276
16:25:36.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.767743 26639 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:25:36.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.781525 26639 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:25:36.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.781857 26639 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:25:36.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.782196 26639 ts_tablet_manager.cc:616] Registered 0 tablets
16:25:36.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:36.782449 26639 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:25:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.390978 26777 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.173.1:42067 every 8 connection(s)
16:25:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.391520 26639 rpc_server.cc:307] RPC server started. Bound to: 127.25.173.1:42067
16:25:37.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.395241 26639 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-0/data/info.pb
16:25:37.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.397329 26292 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 26639
16:25:37.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.397861 26292 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-0/wal/instance
16:25:37.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.173.2@KRBTEST.COM; defaulting to no policy
16:25:37.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:37.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.173.2@KRBTEST.COM" created.
16:25:37.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:37 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764736, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.1@KRBTEST.COM for kudu/127.25.173.60@KRBTEST.COM
16:25:37.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:37 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764736, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.1@KRBTEST.COM for kudu/127.25.173.61@KRBTEST.COM
16:25:37.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:37 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764736, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.1@KRBTEST.COM for kudu/127.25.173.62@KRBTEST.COM
16:25:37.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.533277 26778 heartbeater.cc:344] Connected to a master server at 127.25.173.60:36837
16:25:37.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.533819 26778 heartbeater.cc:461] Registering TS with master...
16:25:37.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.535290 26778 heartbeater.cc:507] Master 127.25.173.60:36837 requested a full tablet report, sending...
16:25:37.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.538573 26533 ts_manager.cc:194] Registered new tserver with Master: 3be9c2934e634e9db2674aa1fcd019af (127.25.173.1:42067)
16:25:37.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.542297 26533 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.25.173.1@KRBTEST.COM'} at 127.25.173.1:60809
16:25:37.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.545398 26780 heartbeater.cc:344] Connected to a master server at 127.25.173.61:39669
16:25:37.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.545780 26780 heartbeater.cc:461] Registering TS with master...
16:25:37.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.546844 26780 heartbeater.cc:507] Master 127.25.173.61:39669 requested a full tablet report, sending...
16:25:37.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.551543 26446 ts_manager.cc:194] Registered new tserver with Master: 3be9c2934e634e9db2674aa1fcd019af (127.25.173.1:42067)
16:25:37.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.552050 26783 heartbeater.cc:344] Connected to a master server at 127.25.173.62:36591
16:25:37.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.552409 26783 heartbeater.cc:461] Registering TS with master...
16:25:37.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.553337 26783 heartbeater.cc:507] Master 127.25.173.62:36591 requested a full tablet report, sending...
16:25:37.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:37.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.173.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:37.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.173.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:37.556972 26292 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.25.173.2: real 0.148s user 0.000s sys 0.014s
16:25:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.556928 26362 ts_manager.cc:194] Registered new tserver with Master: 3be9c2934e634e9db2674aa1fcd019af (127.25.173.1:42067)
16:25:37.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.25.173.2@KRBTEST.COM; defaulting to no policy
16:25:37.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:37.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.25.173.2@KRBTEST.COM" created.
16:25:37.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:37.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.25.173.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:37.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.25.173.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:37.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:37.617480 26292 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:37.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:37.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-1/wal
16:25:37.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-1/data
16:25:37.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:25:37.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:25:37.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:25:37.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:37.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:37.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:37.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-1/logs
16:25:37.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-1/data/info.pb
16:25:37.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:37.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:25:37.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:25:37.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:25:37.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:25:37.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:37.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:37.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:37.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:25:37.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:25:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.2:0
16:25:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.173.2
16:25:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.2
16:25:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.2
16:25:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:25:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:25:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:25:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:25:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:38.062178 26792 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:25:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:38.062740 26792 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:25:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:38.063395 26792 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:25:38.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:38.097770 26792 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:25:38.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:38.099174 26792 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.25.173.2
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:38.138594 26792 tablet_server_runner.cc:78] Tablet server non-default flags:
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-1/data
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-1/wal
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.2
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.2:0
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-1/data/info.pb
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.2
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:25:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:25:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:25:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.173.2
16:25:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-1/logs
16:25:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:25:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:25:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:25:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:25:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:25:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:25:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:25:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:25:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:25:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:38.140012 26792 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:25:38.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:38.141222 26792 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:25:38.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:38.154681 26798 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:38 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764738, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:38.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:38.355273 26425 init.cc:295] Successfully reacquired a new kerberos TGT
16:25:38.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:38.551440 26778 heartbeater.cc:499] Master 127.25.173.60:36837 was elected leader, sending a full tablet report...
16:25:39.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:38.160809 26801 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:39.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:38.157670 26799 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:39.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:39.615576 26800 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1454 milliseconds
16:25:39.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.615692 26792 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:25:39.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.616925 26792 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:25:39.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.619786 26792 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:25:39.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.621809 26792 hybrid_clock.cc:648] HybridClock initialized: now 1773764739621579 us; error 235 us; skew 500 ppm
16:25:39.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.623163 26792 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:25:39.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:39 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764739, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.630890 26792 init.cc:377] Logged in from keytab as kudu/127.25.173.2@KRBTEST.COM (short username kudu)
16:25:39.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.637136 26792 webserver.cc:492] Webserver started at http://127.25.173.2:42087/ using document root <none> and password file <none>
16:25:39.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.638062 26792 fs_manager.cc:362] Metadata directory not provided
16:25:39.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.638291 26792 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:25:39.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.638887 26792 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:25:39.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.643679 26792 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-1/data/instance:
16:25:39.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "212c40dfb16a4399b4215244f669ef7a"
16:25:39.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:39 on dist-test-slave-jdhl"
16:25:39.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.644953 26792 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-1/wal/instance:
16:25:39.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "212c40dfb16a4399b4215244f669ef7a"
16:25:39.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:39 on dist-test-slave-jdhl"
16:25:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.654233 26792 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.001s sys 0.008s
16:25:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.660913 26810 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:25:39.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.662314 26792 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
16:25:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.662701 26792 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-1/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-1/wal
16:25:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "212c40dfb16a4399b4215244f669ef7a"
16:25:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:39 on dist-test-slave-jdhl"
16:25:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.663204 26792 fs_report.cc:389] FS layout report
16:25:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:25:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-1/wal
16:25:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-1/wal
16:25:39.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-1/data/data
16:25:39.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:25:39.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:25:39.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:25:39.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:25:39.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:25:39.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:25:39.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:25:39.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:25:39.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:25:39.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:25:39.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:25:39.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.729480 26792 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:25:39.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.736905 26792 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:25:39.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.737355 26792 kserver.cc:163] Server-wide thread pool size limit: 3276
16:25:39.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.741487 26792 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:25:39.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.748040 26792 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:25:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.748371 26792 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:25:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.748653 26792 ts_tablet_manager.cc:616] Registered 0 tablets
16:25:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.748855 26792 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:25:39.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.956454 26792 rpc_server.cc:307] RPC server started. Bound to: 127.25.173.2:43715
16:25:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.956743 26923 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.173.2:43715 every 8 connection(s)
16:25:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.960075 26792 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-1/data/info.pb
16:25:39.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.971356 26292 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 26792
16:25:39.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:39.972573 26292 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-1/wal/instance
16:25:40.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.173.3@KRBTEST.COM; defaulting to no policy
16:25:40.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:40.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.173.3@KRBTEST.COM" created.
16:25:40.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:40 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764739, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.2@KRBTEST.COM for kudu/127.25.173.60@KRBTEST.COM
16:25:40.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:40 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764739, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.2@KRBTEST.COM for kudu/127.25.173.62@KRBTEST.COM
16:25:40.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:40 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764739, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.2@KRBTEST.COM for kudu/127.25.173.61@KRBTEST.COM
16:25:40.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:40.069677 26924 heartbeater.cc:344] Connected to a master server at 127.25.173.60:36837
16:25:40.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:40.070370 26924 heartbeater.cc:461] Registering TS with master...
16:25:40.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:40.072333 26924 heartbeater.cc:507] Master 127.25.173.60:36837 requested a full tablet report, sending...
16:25:40.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:40.076409 26926 heartbeater.cc:344] Connected to a master server at 127.25.173.62:36591
16:25:40.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:40.076853 26926 heartbeater.cc:461] Registering TS with master...
16:25:40.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:40.076596 26533 ts_manager.cc:194] Registered new tserver with Master: 212c40dfb16a4399b4215244f669ef7a (127.25.173.2:43715)
16:25:40.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:40.078141 26926 heartbeater.cc:507] Master 127.25.173.62:36591 requested a full tablet report, sending...
16:25:40.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:40.079418 26925 heartbeater.cc:344] Connected to a master server at 127.25.173.61:39669
16:25:40.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:40.079870 26925 heartbeater.cc:461] Registering TS with master...
16:25:40.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:40.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.173.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:40.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.173.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:40.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:40.080914 26925 heartbeater.cc:507] Master 127.25.173.61:39669 requested a full tablet report, sending...
16:25:40.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:40.081350 26362 ts_manager.cc:194] Registered new tserver with Master: 212c40dfb16a4399b4215244f669ef7a (127.25.173.2:43715)
16:25:40.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:40.082016 26533 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.25.173.2@KRBTEST.COM'} at 127.25.173.2:52383
16:25:40.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:40.084102 26446 ts_manager.cc:194] Registered new tserver with Master: 212c40dfb16a4399b4215244f669ef7a (127.25.173.2:43715)
16:25:40.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.25.173.3@KRBTEST.COM; defaulting to no policy
16:25:40.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:40.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.25.173.3@KRBTEST.COM" created.
16:25:40.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:25:40.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.25.173.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:40.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.25.173.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab.
16:25:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:40.128999 26292 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-2/wal
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-2/data
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-2/logs
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-2/data/info.pb
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.3:0
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.173.3
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.3
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.3
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:25:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:40.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:40.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:40.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:40.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:25:40.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:40.507731 26938 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:25:40.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:40.508329 26938 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:25:40.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:40.509095 26938 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:25:40.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:40.543726 26938 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:25:40.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:40.545061 26938 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.25.173.3
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:40.592796 26938 tablet_server_runner.cc:78] Tablet server non-default flags:
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-2/data
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-2/wal
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.3
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.3:0
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-2/data/info.pb
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.3
16:25:40.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:25:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:25:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:25:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:25:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.173.3
16:25:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-2/logs
16:25:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:25:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:25:40.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:25:40.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:25:40.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:25:40.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:25:40.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:25:40.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:25:40.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:25:40.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:40.594487 26938 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:25:40.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:40.595780 26938 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:25:40.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:40.610242 26945 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:41.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:41.087965 26924 heartbeater.cc:499] Master 127.25.173.60:36837 was elected leader, sending a full tablet report...
16:25:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:41 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764741, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:41.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:41.232399 26512 init.cc:295] Successfully reacquired a new kerberos TGT
16:25:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:42.022377 26944 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 26938
16:25:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:42.432075 26944 kernel_stack_watchdog.cc:198] Thread 26938 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 406ms:
16:25:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:25:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:25:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:25:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:25:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:40.613054 26946 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:42.433189 26938 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.820s user 0.565s sys 1.131s
16:25:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:42.433530 26938 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.821s user 0.565s sys 1.131s
16:25:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:42.435861 26949 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:42.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:42.440361 26947 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1827 milliseconds
16:25:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.440454 26938 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:25:42.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.442324 26938 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:25:42.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.445235 26938 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:25:42.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.448856 26938 hybrid_clock.cc:648] HybridClock initialized: now 1773764742448835 us; error 62 us; skew 500 ppm
16:25:42.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.449710 26938 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:25:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:42 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764742, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.457075 26938 init.cc:377] Logged in from keytab as kudu/127.25.173.3@KRBTEST.COM (short username kudu)
16:25:42.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.462713 26938 webserver.cc:492] Webserver started at http://127.25.173.3:42897/ using document root <none> and password file <none>
16:25:42.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.463757 26938 fs_manager.cc:362] Metadata directory not provided
16:25:42.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.464015 26938 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:25:42.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.464540 26938 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:25:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.469206 26938 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-2/data/instance:
16:25:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac2de3172b094e988002346949fa14f1"
16:25:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:42 on dist-test-slave-jdhl"
16:25:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.470512 26938 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-2/wal/instance:
16:25:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac2de3172b094e988002346949fa14f1"
16:25:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:42 on dist-test-slave-jdhl"
16:25:42.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.479154 26938 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
16:25:42.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.485327 26961 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:25:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.486611 26938 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
16:25:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.487088 26938 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-2/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-2/wal
16:25:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac2de3172b094e988002346949fa14f1"
16:25:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:42 on dist-test-slave-jdhl"
16:25:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.487474 26938 fs_report.cc:389] FS layout report
16:25:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:25:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-2/wal
16:25:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-2/wal
16:25:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-2/data/data
16:25:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:25:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:25:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:25:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:25:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:25:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:25:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:25:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:25:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:25:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:25:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:25:42.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.559980 26938 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:25:42.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.572816 26938 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:25:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.573730 26938 kserver.cc:163] Server-wide thread pool size limit: 3276
16:25:42.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.576678 26938 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:25:42.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.583768 26938 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:25:42.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.583985 26938 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:25:42.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.584178 26938 ts_tablet_manager.cc:616] Registered 0 tablets
16:25:42.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.584317 26938 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:25:42.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.785724 26938 rpc_server.cc:307] RPC server started. Bound to: 127.25.173.3:37197
16:25:42.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.785809 27074 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.173.3:37197 every 8 connection(s)
16:25:42.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.789060 26938 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-2/data/info.pb
16:25:42.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.795893 26292 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 26938
16:25:42.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.796396 26292 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/ts-2/wal/instance
16:25:42.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:42 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764742, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.3@KRBTEST.COM for kudu/127.25.173.60@KRBTEST.COM
16:25:42.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:42 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764742, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.3@KRBTEST.COM for kudu/127.25.173.61@KRBTEST.COM
16:25:42.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:42 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764742, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.3@KRBTEST.COM for kudu/127.25.173.62@KRBTEST.COM
16:25:42.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.939244 27075 heartbeater.cc:344] Connected to a master server at 127.25.173.60:36837
16:25:42.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.939754 27075 heartbeater.cc:461] Registering TS with master...
16:25:42.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.941648 27075 heartbeater.cc:507] Master 127.25.173.60:36837 requested a full tablet report, sending...
16:25:42.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.944276 27076 heartbeater.cc:344] Connected to a master server at 127.25.173.61:39669
16:25:42.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.944635 27076 heartbeater.cc:461] Registering TS with master...
16:25:42.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.944553 26533 ts_manager.cc:194] Registered new tserver with Master: ac2de3172b094e988002346949fa14f1 (127.25.173.3:37197)
16:25:42.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.945662 27076 heartbeater.cc:507] Master 127.25.173.61:39669 requested a full tablet report, sending...
16:25:42.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.946456 27077 heartbeater.cc:344] Connected to a master server at 127.25.173.62:36591
16:25:42.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.946488 26533 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.25.173.3@KRBTEST.COM'} at 127.25.173.3:34629
16:25:42.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.946841 27077 heartbeater.cc:461] Registering TS with master...
16:25:42.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.948249 27077 heartbeater.cc:507] Master 127.25.173.62:36591 requested a full tablet report, sending...
16:25:42.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.948323 26446 ts_manager.cc:194] Registered new tserver with Master: ac2de3172b094e988002346949fa14f1 (127.25.173.3:37197)
16:25:42.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.950416 26362 ts_manager.cc:194] Registered new tserver with Master: ac2de3172b094e988002346949fa14f1 (127.25.173.3:37197)
16:25:42.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:42.968170 26292 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
16:25:42.983 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:25:42.984 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
16:25:42.992 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
16:25:42.993 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
16:25:42.998 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.25.173.62"
port: 36591
}
bound_http_address {
host: "127.25.173.62"
port: 41617
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.25.173.61"
port: 39669
}
bound_http_address {
host: "127.25.173.61"
port: 46803
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.25.173.60"
port: 36837
}
bound_http_address {
host: "127.25.173.60"
port: 41079
}
}
}
16:25:42.998 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:25:42.999 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.25.173.1"
port: 42067
}
bound_http_address {
host: "127.25.173.1"
port: 34185
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.25.173.2"
port: 43715
}
bound_http_address {
host: "127.25.173.2"
port: 42087
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.25.173.3"
port: 37197
}
bound_http_address {
host: "127.25.173.3"
port: 42897
}
}
}
16:25:42.999 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:25:43.000 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:25:43.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:43 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764743, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:43.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:25:43.037 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:25:43.040 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5cc
16:25:43.050 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:25:43.050 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5cc
16:25:43.051 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:25:43.076 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
16:25:43.078 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:25:43.078 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:25:43.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:43 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764743, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:43.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:25:43.113 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:25:43.113 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5cc
16:25:43.115 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:25:43.115 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.173.60:36837
16:25:43.116 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
16:25:43.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:43.117621 26292 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 26492
16:25:43.145 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:25:43.146 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.173.61:39669
16:25:43.146 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
16:25:43.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:43.146540 26292 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 26406
16:25:43.179 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:25:43.180 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.173.62:36591
16:25:43.180 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
16:25:43.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:43.181151 26292 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 26326
16:25:43.215 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:25:43.216 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.173.60:36837
16:25:43.220 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
16:25:43.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:43.222265 26292 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:43.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:43.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/wal
16:25:43.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data
16:25:43.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:25:43.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:25:43.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:25:43.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:43.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/logs
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data/info.pb
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.60:36837
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.60
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41079
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.60
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:43.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:43.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:43.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:43.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:43.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33477/jwks.json
16:25:43.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:25:43.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:25:43.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:43.564189 27089 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:25:43.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:43.564733 27089 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:25:43.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:43.565356 27089 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:25:43.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:43.587325 26783 heartbeater.cc:646] Failed to heartbeat to 127.25.173.62:36591 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.25.173.62:36591: connect: Connection refused (error 111)
16:25:43.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:43.604118 27089 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:25:43.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:43.604477 27089 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:25:43.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:43.604933 27089 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:25:43.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:43.605203 27089 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:25:43.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:43.605408 27089 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:25:43.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:43.605602 27089 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:25:43.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:43.605754 27089 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33477/jwks.json
16:25:43.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:43.645006 27089 master_runner.cc:386] Master server non-default flags:
16:25:43.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:43.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:43.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:43.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:43.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data
16:25:43.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/wal
16:25:43.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:25:43.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:43.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:43.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.60
16:25:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.60:36837
16:25:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:25:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33477/jwks.json
16:25:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:25:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:25:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data/info.pb
16:25:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.60
16:25:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41079
16:25:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:25:43.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:43.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:25:43.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:25:43.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:43.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/logs
16:25:43.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:43.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:25:43.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:25:43.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:25:43.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:25:43.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:25:43.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:25:43.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:25:43.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:25:43.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:25:43.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:43.646363 27089 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:25:43.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:43.647396 27089 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:25:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:43.661267 27100 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:43.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:43.952847 27075 heartbeater.cc:499] Master 127.25.173.60:36837 was elected leader, sending a full tablet report...
16:25:44.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:44.006286 27077 heartbeater.cc:646] Failed to heartbeat to 127.25.173.62:36591 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.25.173.62:36591: connect: Connection refused (error 111)
16:25:44.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:44.116758 26925 heartbeater.cc:646] Failed to heartbeat to 127.25.173.61:39669 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.25.173.61:39669: connect: Connection refused (error 111)
16:25:45.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:45.065991 27099 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 27089
16:25:45.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:45.500562 27089 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.840s user 0.675s sys 1.059s
16:25:45.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:45.500931 27089 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.840s user 0.675s sys 1.059s
16:25:45.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:43.661896 27101 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:45.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:45.503367 27103 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:45.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:45.506203 27102 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1840 milliseconds
16:25:45.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.506311 27089 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:25:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.507719 27089 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:25:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.510365 27089 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:25:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.511843 27089 hybrid_clock.cc:648] HybridClock initialized: now 1773764745511792 us; error 51 us; skew 500 ppm
16:25:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.512861 27089 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:25:45.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:45 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764745, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:45.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.519930 27089 init.cc:377] Logged in from keytab as kudu/127.25.173.60@KRBTEST.COM (short username kudu)
16:25:45.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.525665 27089 webserver.cc:492] Webserver started at http://127.25.173.60:41079/ using document root <none> and password file <none>
16:25:45.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.526983 27089 fs_manager.cc:362] Metadata directory not provided
16:25:45.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.527292 27089 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:25:45.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.535924 27089 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
16:25:45.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.541064 27123 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:25:45.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.542290 27089 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
16:25:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.542690 27089 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/wal
16:25:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7724600eba9e4ebea1a0a1f3720ac88c"
16:25:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:31 on dist-test-slave-jdhl"
16:25:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.544828 27089 fs_report.cc:389] FS layout report
16:25:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:25:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/wal
16:25:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/wal
16:25:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data/data
16:25:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:25:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:25:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:25:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:25:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:25:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:25:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:25:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:25:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:25:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:25:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:25:45.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.603581 27089 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:25:45.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.609557 27089 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:25:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.610091 27089 kserver.cc:163] Server-wide thread pool size limit: 3276
16:25:45.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.691735 27089 rpc_server.cc:307] RPC server started. Bound to: 127.25.173.60:36837
16:25:45.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.691833 27176 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.173.60:36837 every 8 connection(s)
16:25:45.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.694902 27089 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data/info.pb
16:25:45.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.699931 26292 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 27089
16:25:45.700 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:25:45.701 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.173.61:39669
16:25:45.701 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
16:25:45.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.702903 26292 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:45.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:45.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/wal
16:25:45.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data
16:25:45.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:25:45.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:25:45.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:25:45.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:45.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:45.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:45.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/logs
16:25:45.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data/info.pb
16:25:45.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:25:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:25:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:25:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:25:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:25:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:25:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.61:39669
16:25:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.61
16:25:45.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46803
16:25:45.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:45.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.61
16:25:45.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:45.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:25:45.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:25:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:25:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33477/jwks.json
16:25:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:25:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:25:45.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.711205 27177 sys_catalog.cc:263] Verifying existing consensus state
16:25:45.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.719568 27177 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: Bootstrap starting.
16:25:45.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.775306 27177 log.cc:826] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: Log is configured to *not* fsync() on all Append() calls
16:25:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.793329 27177 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:25:45.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.794272 27177 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: Bootstrap complete.
16:25:45.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.822110 27177 raft_consensus.cc:359] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:45.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.824676 27177 raft_consensus.cc:740] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7724600eba9e4ebea1a0a1f3720ac88c, State: Initialized, Role: FOLLOWER
16:25:45.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.825742 27177 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:45.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.829020 27182 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:45.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.829883 27182 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: This master's current role is: FOLLOWER
16:25:45.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.831007 27177 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: configured and running, proceeding with master startup.
16:25:45.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.865061 27193 catalog_manager.cc:1269] Loaded cluster ID: 723802fca27f462db7efb81f8e18a050
16:25:45.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.865934 27193 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: loading cluster ID for follower catalog manager: success
16:25:45.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.875259 27193 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: acquiring CA information for follower catalog manager: success
16:25:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.882108 27193 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:25:45.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.860733 27089 master_runner.cc:186] Error getting master registration for 127.25.173.62:36591: Network error: Client connection negotiation failed: client connection to 127.25.173.62:36591: connect: Connection refused (error 111),
16:25:45.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:45.892483 27089 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:25:46.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:46.067737 27075 heartbeater.cc:344] Connected to a master server at 127.25.173.60:36837
16:25:46.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:46.068321 27075 heartbeater.cc:499] Master 127.25.173.60:36837 was elected leader, sending a full tablet report...
16:25:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:46.075208 27142 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ac2de3172b094e988002346949fa14f1" instance_seqno: 1773764742734114) as {username='kudu', principal='kudu/127.25.173.3@KRBTEST.COM'} at 127.25.173.3:48329; Asking this server to re-register.
16:25:46.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:46.077015 27075 heartbeater.cc:461] Registering TS with master...
16:25:46.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:46.080821 27142 ts_manager.cc:194] Registered new tserver with Master: ac2de3172b094e988002346949fa14f1 (127.25.173.3:37197)
16:25:46.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:46.105264 27179 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:25:46.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:46.105990 27179 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:25:46.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:46.106876 27179 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:25:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:46.142926 27179 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:25:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:46.143210 27179 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:25:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:46.143669 27179 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:25:46.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:46.144025 27179 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:25:46.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:46.144359 27179 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:25:46.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:46.144760 27179 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:25:46.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:46.145032 27179 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33477/jwks.json
16:25:46.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:46.177922 26924 heartbeater.cc:344] Connected to a master server at 127.25.173.60:36837
16:25:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:46.180650 27142 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "212c40dfb16a4399b4215244f669ef7a" instance_seqno: 1773764739894621) as {username='kudu', principal='kudu/127.25.173.2@KRBTEST.COM'} at 127.25.173.2:35493; Asking this server to re-register.
16:25:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:46.182003 26924 heartbeater.cc:461] Registering TS with master...
16:25:46.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:46.185832 27142 ts_manager.cc:194] Registered new tserver with Master: 212c40dfb16a4399b4215244f669ef7a (127.25.173.2:43715)
16:25:46.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:46.201299 27179 master_runner.cc:386] Master server non-default flags:
16:25:46.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data
16:25:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/wal
16:25:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:25:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.61
16:25:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.61:39669
16:25:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33477/jwks.json
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data/info.pb
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.61
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46803
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/logs
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:25:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:25:46.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:25:46.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:25:46.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:25:46.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:25:46.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:46.202956 27179 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:25:46.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:46.204458 27179 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:25:46.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:46.218528 27207 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:46.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:46 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764746, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:46.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:46.458791 26660 init.cc:295] Successfully reacquired a new kerberos TGT
16:25:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:46.660894 26778 heartbeater.cc:344] Connected to a master server at 127.25.173.60:36837
16:25:46.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:46.664340 27142 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "3be9c2934e634e9db2674aa1fcd019af" instance_seqno: 1773764737282488) as {username='kudu', principal='kudu/127.25.173.1@KRBTEST.COM'} at 127.25.173.1:36161; Asking this server to re-register.
16:25:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:46.665947 26778 heartbeater.cc:461] Registering TS with master...
16:25:46.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:46.669693 27142 ts_manager.cc:194] Registered new tserver with Master: 3be9c2934e634e9db2674aa1fcd019af (127.25.173.1:42067)
16:25:47.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:47.014505 27216 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:25:47.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:47.015504 27216 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:47.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:47.111158 27216 leader_election.cc:290] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9bddceeba6654533b175280b4491784f (127.25.173.62:36591), 9555719ea49a4994a3973eb4452a45df (127.25.173.61:39669)
16:25:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:47.127945 27125 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.173.61:39669: connect: Connection refused (error 111)
16:25:47.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:47.143702 27125 leader_election.cc:336] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9555719ea49a4994a3973eb4452a45df (127.25.173.61:39669): Network error: Client connection negotiation failed: client connection to 127.25.173.61:39669: connect: Connection refused (error 111)
16:25:47.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:47.151927 27127 leader_election.cc:336] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9bddceeba6654533b175280b4491784f (127.25.173.62:36591): Network error: Client connection negotiation failed: client connection to 127.25.173.62:36591: connect: Connection refused (error 111)
16:25:47.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:47.152699 27127 leader_election.cc:304] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7724600eba9e4ebea1a0a1f3720ac88c; no voters: 9555719ea49a4994a3973eb4452a45df, 9bddceeba6654533b175280b4491784f
16:25:47.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:47.154485 27216 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
16:25:48.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:47.621490 27206 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 27179
16:25:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:48.254261 27206 kernel_stack_watchdog.cc:198] Thread 27179 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 392ms:
16:25:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:25:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:25:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:25:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:25:48.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:48.257002 27179 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.031s user 0.770s sys 1.069s
16:25:48.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:46.226737 27208 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:48.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:48.259827 27179 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.034s user 0.770s sys 1.070s
16:25:48.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:48.262696 27210 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:48.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:48.266120 27209 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2033 milliseconds
16:25:48.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.266181 27179 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:25:48.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.267715 27179 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:25:48.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.271448 27179 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:25:48.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.275095 27179 hybrid_clock.cc:648] HybridClock initialized: now 1773764748274823 us; error 274 us; skew 500 ppm
16:25:48.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.275977 27179 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:25:48.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:48 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764748, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:48.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.283483 27179 init.cc:377] Logged in from keytab as kudu/127.25.173.61@KRBTEST.COM (short username kudu)
16:25:48.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.294656 27179 webserver.cc:492] Webserver started at http://127.25.173.61:46803/ using document root <none> and password file <none>
16:25:48.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.295763 27179 fs_manager.cc:362] Metadata directory not provided
16:25:48.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.296057 27179 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:25:48.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.305085 27179 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.001s sys 0.005s
16:25:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.310534 27235 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:25:48.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.311899 27179 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
16:25:48.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.312314 27179 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/wal
16:25:48.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9555719ea49a4994a3973eb4452a45df"
16:25:48.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:28 on dist-test-slave-jdhl"
16:25:48.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.315140 27179 fs_report.cc:389] FS layout report
16:25:48.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:25:48.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/wal
16:25:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/wal
16:25:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data/data
16:25:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:25:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:25:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:25:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:25:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:25:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:25:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:25:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:25:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:25:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:25:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:25:48.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.395241 27179 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:25:48.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.407612 27179 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:25:48.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.408066 27179 kserver.cc:163] Server-wide thread pool size limit: 3276
16:25:48.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.498979 27179 rpc_server.cc:307] RPC server started. Bound to: 127.25.173.61:39669
16:25:48.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.499110 27286 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.173.61:39669 every 8 connection(s)
16:25:48.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.502152 27179 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data/info.pb
16:25:48.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.502919 26292 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 27179
16:25:48.504 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:25:48.504 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.173.62:36591
16:25:48.504 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
16:25:48.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.506384 26292 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/wal
16:25:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data
16:25:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:25:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:25:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:25:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/logs
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data/info.pb
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.62:36591
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.62
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41617
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.62
16:25:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:25:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:25:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:25:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33477/jwks.json
16:25:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:25:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:25:48.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.525475 27287 sys_catalog.cc:263] Verifying existing consensus state
16:25:48.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.531952 27287 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: Bootstrap starting.
16:25:48.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.578451 27287 log.cc:826] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: Log is configured to *not* fsync() on all Append() calls
16:25:48.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.600420 27287 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:25:48.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.601361 27287 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: Bootstrap complete.
16:25:48.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.627066 27287 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:48.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.630488 27287 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9555719ea49a4994a3973eb4452a45df, State: Initialized, Role: FOLLOWER
16:25:48.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.631688 27287 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:48.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.635367 27291 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.636603 27291 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: This master's current role is: FOLLOWER
16:25:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.637331 27287 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: configured and running, proceeding with master startup.
16:25:48.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.746932 27302 catalog_manager.cc:1269] Loaded cluster ID: 723802fca27f462db7efb81f8e18a050
16:25:48.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.747607 27302 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: loading cluster ID for follower catalog manager: success
16:25:48.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.756929 27302 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: acquiring CA information for follower catalog manager: success
16:25:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.764730 27302 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:25:48.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:48 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764746, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.1@KRBTEST.COM for kudu/127.25.173.61@KRBTEST.COM
16:25:48.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.796905 27308 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:25:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.797401 27308 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:48.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.751169 27179 master_runner.cc:186] Error getting master registration for 127.25.173.62:36591: Network error: Client connection negotiation failed: client connection to 127.25.173.62:36591: connect: Connection refused (error 111),
16:25:48.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.809405 27179 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:25:48.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.821524 27308 leader_election.cc:290] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9bddceeba6654533b175280b4491784f (127.25.173.62:36591), 9555719ea49a4994a3973eb4452a45df (127.25.173.61:39669)
16:25:48.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.847808 26780 heartbeater.cc:344] Connected to a master server at 127.25.173.61:39669
16:25:48.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:48.854388 27127 leader_election.cc:336] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9bddceeba6654533b175280b4491784f (127.25.173.62:36591): Network error: Client connection negotiation failed: client connection to 127.25.173.62:36591: connect: Connection refused (error 111)
16:25:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.854514 27252 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "3be9c2934e634e9db2674aa1fcd019af" instance_seqno: 1773764737282488) as {username='kudu', principal='kudu/127.25.173.1@KRBTEST.COM'} at 127.25.173.1:59087; Asking this server to re-register.
16:25:48.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.856284 26780 heartbeater.cc:461] Registering TS with master...
16:25:48.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.859463 27262 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "9555719ea49a4994a3973eb4452a45df" is_pre_election: true
16:25:48.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.860422 27262 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7724600eba9e4ebea1a0a1f3720ac88c in term 1.
16:25:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.860814 27252 ts_manager.cc:194] Registered new tserver with Master: 3be9c2934e634e9db2674aa1fcd019af (127.25.173.1:42067)
16:25:48.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.861852 27125 leader_election.cc:304] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7724600eba9e4ebea1a0a1f3720ac88c, 9555719ea49a4994a3973eb4452a45df; no voters: 9bddceeba6654533b175280b4491784f
16:25:48.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.863040 27308 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 1 FOLLOWER]: Leader pre-election won for term 2
16:25:48.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.863382 27308 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:25:48.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.863713 27308 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 1 FOLLOWER]: Advancing to term 2
16:25:48.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.875128 27308 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:48.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:48.882117 27127 leader_election.cc:336] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 9bddceeba6654533b175280b4491784f (127.25.173.62:36591): Network error: Client connection negotiation failed: client connection to 127.25.173.62:36591: connect: Connection refused (error 111)
16:25:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.884079 27308 leader_election.cc:290] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 2 election: Requested vote from peers 9bddceeba6654533b175280b4491784f (127.25.173.62:36591), 9555719ea49a4994a3973eb4452a45df (127.25.173.61:39669)
16:25:48.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.884984 27262 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "9555719ea49a4994a3973eb4452a45df"
16:25:48.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.885591 27262 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 1 FOLLOWER]: Advancing to term 2
16:25:48.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.896100 27262 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7724600eba9e4ebea1a0a1f3720ac88c in term 2.
16:25:48.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.897295 27125 leader_election.cc:304] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7724600eba9e4ebea1a0a1f3720ac88c, 9555719ea49a4994a3973eb4452a45df; no voters: 9bddceeba6654533b175280b4491784f
16:25:48.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.898473 27308 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 2 FOLLOWER]: Leader election won for term 2
16:25:48.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.913713 27308 raft_consensus.cc:697] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 2 LEADER]: Becoming Leader. State: Replica: 7724600eba9e4ebea1a0a1f3720ac88c, State: Running, Role: LEADER
16:25:48.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.914871 27308 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:48.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.932603 27313 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7724600eba9e4ebea1a0a1f3720ac88c. Latest consensus state: current_term: 2 leader_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:48.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.933813 27313 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: This master's current role is: LEADER
16:25:48.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.946308 27315 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
16:25:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.970289 27315 catalog_manager.cc:1494] Initializing Kudu cluster ID...
16:25:48.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.981127 27315 catalog_manager.cc:1269] Loaded cluster ID: 723802fca27f462db7efb81f8e18a050
16:25:48.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.981789 27315 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
16:25:48.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.987690 27315 catalog_manager.cc:1514] Loading token signing keys...
16:25:48.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.992635 27315 catalog_manager.cc:6055] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: Loaded TSK: 0
16:25:48.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:48.994256 27315 catalog_manager.cc:1524] Initializing in-progress tserver states...
16:25:49.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:49.134447 27289 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:25:49.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:49.135658 27289 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:25:49.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:49.137454 27289 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:25:49.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.155730 27076 heartbeater.cc:344] Connected to a master server at 127.25.173.61:39669
16:25:49.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.158593 27252 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ac2de3172b094e988002346949fa14f1" instance_seqno: 1773764742734114) as {username='kudu', principal='kudu/127.25.173.3@KRBTEST.COM'} at 127.25.173.3:48565; Asking this server to re-register.
16:25:49.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.163192 27076 heartbeater.cc:461] Registering TS with master...
16:25:49.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.166553 27252 ts_manager.cc:194] Registered new tserver with Master: ac2de3172b094e988002346949fa14f1 (127.25.173.3:37197)
16:25:49.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:49.223189 27289 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:25:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:49.223531 27289 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:25:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:49.224016 27289 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:25:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:49.224359 27289 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:25:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:49.224696 27289 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:25:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:49.225028 27289 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:25:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:49.225286 27289 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33477/jwks.json
16:25:49.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.239463 26925 heartbeater.cc:344] Connected to a master server at 127.25.173.61:39669
16:25:49.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.243104 27252 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "212c40dfb16a4399b4215244f669ef7a" instance_seqno: 1773764739894621) as {username='kudu', principal='kudu/127.25.173.2@KRBTEST.COM'} at 127.25.173.2:39503; Asking this server to re-register.
16:25:49.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.245910 26925 heartbeater.cc:461] Registering TS with master...
16:25:49.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.249696 27252 ts_manager.cc:194] Registered new tserver with Master: 212c40dfb16a4399b4215244f669ef7a (127.25.173.2:43715)
16:25:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.309772 27289 master_runner.cc:386] Master server non-default flags:
16:25:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data
16:25:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/wal
16:25:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:25:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.62
16:25:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.62:36591
16:25:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33477/jwks.json
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data/info.pb
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.62
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41617
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/logs
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:25:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.311542 27289 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:25:49.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.312979 27289 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:25:49.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:49.358393 27326 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:49.445278 27127 consensus_peers.cc:597] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c -> Peer 9bddceeba6654533b175280b4491784f (127.25.173.62:36591): Couldn't send request to peer 9bddceeba6654533b175280b4491784f. Status: Network error: Client connection negotiation failed: client connection to 127.25.173.62:36591: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
16:25:49.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.462739 27262 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 2 FOLLOWER]: Refusing update from remote peer 7724600eba9e4ebea1a0a1f3720ac88c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
16:25:49.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.464730 27313 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [LEADER]: Connected to new peer: Peer: permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
16:25:49.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.500690 27331 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7724600eba9e4ebea1a0a1f3720ac88c. Latest consensus state: current_term: 2 leader_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:49.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.502364 27331 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: This master's current role is: FOLLOWER
16:25:49.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.549803 27313 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:49.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.551164 27313 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: This master's current role is: LEADER
16:25:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.555752 27331 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:49.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.556905 27331 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: This master's current role is: FOLLOWER
16:25:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:49 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764749, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:49.637120 26806 init.cc:295] Successfully reacquired a new kerberos TGT
16:25:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:50.131219 27075 heartbeater.cc:499] Master 127.25.173.60:36837 was elected leader, sending a full tablet report...
16:25:50.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:50.216776 26924 heartbeater.cc:499] Master 127.25.173.60:36837 was elected leader, sending a full tablet report...
16:25:50.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:50.706599 26778 heartbeater.cc:499] Master 127.25.173.60:36837 was elected leader, sending a full tablet report...
16:25:50.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:50.898675 27089 master_runner.cc:186] Error getting master registration for 127.25.173.62:36591: Network error: Client connection negotiation failed: client connection to 127.25.173.62:36591: connect: Connection refused (error 111),
16:25:50.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:50.899338 27089 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:25:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:50.742677 27325 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 27289
16:25:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:49.432358 27327 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:51.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:51.230804 27289 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.819s user 0.618s sys 1.030s
16:25:51.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:51.231251 27289 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.824s user 0.618s sys 1.030s
16:25:51.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.238504 27289 server_base.cc:1060] running on GCE node
16:25:51.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:51.239590 27347 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:51.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.241449 27289 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:25:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.245988 27289 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:25:51.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.249715 27289 hybrid_clock.cc:648] HybridClock initialized: now 1773764751249651 us; error 139 us; skew 500 ppm
16:25:51.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.251159 27289 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:25:51.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:51 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764751, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:51.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.262283 27289 init.cc:377] Logged in from keytab as kudu/127.25.173.62@KRBTEST.COM (short username kudu)
16:25:51.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.268455 27289 webserver.cc:492] Webserver started at http://127.25.173.62:41617/ using document root <none> and password file <none>
16:25:51.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.269748 27289 fs_manager.cc:362] Metadata directory not provided
16:25:51.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.270190 27289 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:25:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.281072 27289 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.002s sys 0.003s
16:25:51.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.287108 27354 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:25:51.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.288530 27289 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
16:25:51.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.288975 27289 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/wal
16:25:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9bddceeba6654533b175280b4491784f"
16:25:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:25 on dist-test-slave-jdhl"
16:25:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.292424 27289 fs_report.cc:389] FS layout report
16:25:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:25:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/wal
16:25:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/wal
16:25:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data/data
16:25:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:25:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:25:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:25:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:25:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:25:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:25:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:25:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:25:51.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:25:51.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:25:51.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:25:51.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.361578 27289 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:25:51.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.370491 27289 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:25:51.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.371199 27289 kserver.cc:163] Server-wide thread pool size limit: 3276
16:25:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.479360 27289 rpc_server.cc:307] RPC server started. Bound to: 127.25.173.62:36591
16:25:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.479480 27405 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.173.62:36591 every 8 connection(s)
16:25:51.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.482388 27289 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data/info.pb
16:25:51.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.489428 26292 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 27289
16:25:51.490 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:25:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.500026 27406 sys_catalog.cc:263] Verifying existing consensus state
16:25:51.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.508445 27406 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: Bootstrap starting.
16:25:51.549 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.173.60:36837(127.25.173.60:36837)] connecting to peer
16:25:51.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.562717 27406 log.cc:826] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: Log is configured to *not* fsync() on all Append() calls
16:25:51.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.588382 27406 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:25:51.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.589481 27406 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: Bootstrap complete.
16:25:51.619 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.173.61:39669(127.25.173.61:39669)] connecting to peer
16:25:51.620 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.173.62:36591(127.25.173.62:36591)] connecting to peer
16:25:51.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.625344 27406 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:51.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.630172 27406 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9bddceeba6654533b175280b4491784f, State: Initialized, Role: FOLLOWER
16:25:51.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.631295 27406 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:51.642 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.25.173.61:39669(127.25.173.61:39669)] Successfully connected to peer
16:25:51.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.644516 27406 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [sys.catalog]: configured and running, proceeding with master startup.
16:25:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.648255 27413 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.649161 27413 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [sys.catalog]: This master's current role is: FOLLOWER
16:25:51.663 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.25.173.60:36837(127.25.173.60:36837)] Successfully connected to peer
16:25:51.728 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.25.173.62:36591(127.25.173.62:36591)] Successfully connected to peer
16:25:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.844520 27429 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
16:25:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:51 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764746, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.1@KRBTEST.COM for kudu/127.25.173.62@KRBTEST.COM
16:25:51.883 [WARN - kudu-nio-0] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
16:25:51.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.911871 26783 heartbeater.cc:344] Connected to a master server at 127.25.173.62:36591
16:25:51.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:51 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764751, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.62@KRBTEST.COM for kudu/127.25.173.62@KRBTEST.COM
16:25:51.915 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.25.173.62
16:25:51.918 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.25.173.61
16:25:51.918 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.25.173.60
16:25:51.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.918527 27371 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "3be9c2934e634e9db2674aa1fcd019af" instance_seqno: 1773764737282488) as {username='kudu', principal='kudu/127.25.173.1@KRBTEST.COM'} at 127.25.173.1:57387; Asking this server to re-register.
16:25:51.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.920544 26783 heartbeater.cc:461] Registering TS with master...
16:25:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:51.924564 27371 ts_manager.cc:194] Registered new tserver with Master: 3be9c2934e634e9db2674aa1fcd019af (127.25.173.1:42067)
16:25:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:51 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764751, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.62@KRBTEST.COM for kudu/127.25.173.61@KRBTEST.COM
16:25:52.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:52 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764751, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.62@KRBTEST.COM for kudu/127.25.173.60@KRBTEST.COM
16:25:52.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:52 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764745, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.60@KRBTEST.COM for kudu/127.25.173.62@KRBTEST.COM
16:25:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:52.088210 27381 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [term 1 FOLLOWER]: Advancing to term 2
16:25:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:52.158358 27413 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7724600eba9e4ebea1a0a1f3720ac88c. Latest consensus state: current_term: 2 leader_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:52.159109 27413 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [sys.catalog]: This master's current role is: FOLLOWER
16:25:52.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:52.161793 27439 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:52.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:52.164863 27439 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f [sys.catalog]: This master's current role is: FOLLOWER
16:25:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:52.228243 27431 negotiation.cc:337] Failed RPC negotiation. Trace:
16:25:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:25:52.215287 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.25.173.3:42961 (local address 127.25.173.62:36591)
16:25:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:25:52.216183 (+ 896us) server_negotiation.cc:207] Beginning negotiation
16:25:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:25:52.216221 (+ 38us) server_negotiation.cc:400] Waiting for connection header
16:25:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:25:52.219158 (+ 2937us) server_negotiation.cc:408] Connection header received
16:25:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:25:52.219655 (+ 497us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
16:25:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:25:52.219684 (+ 29us) server_negotiation.cc:462] Received NEGOTIATE request from client
16:25:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:25:52.219973 (+ 289us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
16:25:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:25:52.220174 (+ 201us) server_negotiation.cc:227] Negotiated authn=SASL
16:25:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:25:52.226612 (+ 6438us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.25.173.3:42961: BlockingRecv error: recv got EOF from 127.25.173.3:42961 (error 108)
16:25:52.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":87}
16:25:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:52 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764749, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.2@KRBTEST.COM for kudu/127.25.173.62@KRBTEST.COM
16:25:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:52.303162 26926 heartbeater.cc:344] Connected to a master server at 127.25.173.62:36591
16:25:52.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:52.305717 27371 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "212c40dfb16a4399b4215244f669ef7a" instance_seqno: 1773764739894621) as {username='kudu', principal='kudu/127.25.173.2@KRBTEST.COM'} at 127.25.173.2:46553; Asking this server to re-register.
16:25:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:52.307545 26926 heartbeater.cc:461] Registering TS with master...
16:25:52.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:52.310305 27371 ts_manager.cc:194] Registered new tserver with Master: 212c40dfb16a4399b4215244f669ef7a (127.25.173.2:43715)
16:25:52.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:52 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773764743, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.173.60@KRBTEST.COM
16:25:52.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:52 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773764743, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.173.61@KRBTEST.COM
16:25:52.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:52 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773764743, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.173.62@KRBTEST.COM
16:25:52.423 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x9f6ca3aa, L:/127.0.0.1:48364 - R:/127.25.173.62:36591] using SASL/GSSAPI
16:25:52.425 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x1575786a, L:/127.0.0.1:46492 - R:/127.25.173.61:39669] using SASL/GSSAPI
16:25:52.431 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x699d55a6, L:/127.0.0.1:43804 - R:/127.25.173.60:36837] using SASL/GSSAPI
16:25:52.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:52 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764752, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:52.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:52.465162 26957 init.cc:295] Successfully reacquired a new kerberos TGT
16:25:52.488 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
16:25:52.490 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=995), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:36837, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:39669, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:36591, [949ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:36591: OK, [950ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:39669: OK, [953ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:36837: OK, deferred=null) after lookup
16:25:52.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:52.566386 27434 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:52.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:52.567597 27434 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: This master's current role is: LEADER
16:25:52.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:52.851640 27429 catalog_manager.cc:1269] Loaded cluster ID: 723802fca27f462db7efb81f8e18a050
16:25:52.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:52.851929 27429 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: loading cluster ID for follower catalog manager: success
16:25:52.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:52.857414 27429 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: acquiring CA information for follower catalog manager: success
16:25:52.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:52.861186 27429 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:25:53.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:53.239749 27077 heartbeater.cc:344] Connected to a master server at 127.25.173.62:36591
16:25:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:53.242504 27371 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ac2de3172b094e988002346949fa14f1" instance_seqno: 1773764742734114) as {username='kudu', principal='kudu/127.25.173.3@KRBTEST.COM'} at 127.25.173.3:59589; Asking this server to re-register.
16:25:53.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:53.244041 27077 heartbeater.cc:461] Registering TS with master...
16:25:53.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:53.247200 27371 ts_manager.cc:194] Registered new tserver with Master: ac2de3172b094e988002346949fa14f1 (127.25.173.3:37197)
16:25:53.538 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:25:53.539 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:25:53.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:53 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764753, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:53.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:25:53.582 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:25:53.583 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5cc
16:25:53.585 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:25:53.585 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.173.60:36837
16:25:53.586 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
16:25:53.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:53.587266 26292 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 27089
16:25:53.617 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:25:53.618 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.173.61:39669
16:25:53.618 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
16:25:53.619 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.25.173.60:36837(127.25.173.60:36837)] handling channelInactive
16:25:53.619 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.60:36837(127.25.173.60:36837)] cleaning up while in state READY due to: connection closed
16:25:53.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:53.619053 26292 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 27179
16:25:53.656 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.25.173.61:39669(127.25.173.61:39669)] handling channelInactive
16:25:53.656 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.61:39669(127.25.173.61:39669)] cleaning up while in state READY due to: connection closed
16:25:53.656 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:25:53.656 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.173.62:36591
16:25:53.657 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
16:25:53.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:53.657454 26292 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 27289
16:25:53.687 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.25.173.62:36591(127.25.173.62:36591)] handling channelInactive
16:25:53.688 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.25.173.62:36591(127.25.173.62:36591)] cleaning up while in state READY due to: connection closed
16:25:53.689 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:25:53.689 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.173.60:36837
16:25:53.689 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
16:25:53.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:53.691216 26292 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:53.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:53.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/wal
16:25:53.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data
16:25:53.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/logs
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data/info.pb
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.60:36837
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.60
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41079
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.60
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33477/jwks.json
16:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:25:54.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:54.048705 27454 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:25:54.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:54.049256 27454 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:25:54.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:54.049872 27454 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:25:54.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:54.084890 27454 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:25:54.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:54.085140 27454 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:25:54.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:54.085629 27454 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:25:54.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:54.085964 27454 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:25:54.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:54.086220 27454 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:25:54.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:54.086457 27454 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:25:54.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:54.086635 27454 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33477/jwks.json
16:25:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:54.125881 27454 master_runner.cc:386] Master server non-default flags:
16:25:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data
16:25:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/wal
16:25:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:25:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.60
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.60:36837
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33477/jwks.json
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data/info.pb
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.60
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41079
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/logs
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:25:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:54.127328 27454 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:25:54.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:54.128334 27454 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:25:54.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:54.140394 27461 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:55.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:54.142230 27462 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:55.575431 27464 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:55.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:55.578642 27463 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1432 milliseconds
16:25:55.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.578749 27454 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:25:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.580091 27454 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:25:55.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.582690 27454 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:25:55.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.584174 27454 hybrid_clock.cc:648] HybridClock initialized: now 1773764755584133 us; error 63 us; skew 500 ppm
16:25:55.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.585093 27454 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:25:55.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:55.589124 25796 debug-util.cc:398] Leaking SignalData structure 0x7b0800082be0 after lost signal to thread 25729
16:25:55.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:55.589932 25796 debug-util.cc:398] Leaking SignalData structure 0x7b08000a75e0 after lost signal to thread 25799
16:25:55.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:55 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764755, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:55.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.592460 27454 init.cc:377] Logged in from keytab as kudu/127.25.173.60@KRBTEST.COM (short username kudu)
16:25:55.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.598182 27454 webserver.cc:492] Webserver started at http://127.25.173.60:41079/ using document root <none> and password file <none>
16:25:55.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.599398 27454 fs_manager.cc:362] Metadata directory not provided
16:25:55.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.599619 27454 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:25:55.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.608675 27454 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.004s sys 0.000s
16:25:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.614233 27478 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:25:55.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.616041 27454 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.000s
16:25:55.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.616497 27454 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/wal
16:25:55.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7724600eba9e4ebea1a0a1f3720ac88c"
16:25:55.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:31 on dist-test-slave-jdhl"
16:25:55.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.619711 27454 fs_report.cc:389] FS layout report
16:25:55.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:25:55.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/wal
16:25:55.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/wal
16:25:55.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data/data
16:25:55.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:25:55.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:25:55.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:25:55.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:25:55.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:25:55.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:25:55.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:25:55.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:25:55.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:25:55.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:25:55.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:25:55.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.720815 27454 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:25:55.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.737172 27454 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:25:55.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.738036 27454 kserver.cc:163] Server-wide thread pool size limit: 3276
16:25:55.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.834177 27454 rpc_server.cc:307] RPC server started. Bound to: 127.25.173.60:36837
16:25:55.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.834268 27530 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.173.60:36837 every 8 connection(s)
16:25:55.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.837451 27454 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-2/data/info.pb
16:25:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.845849 26292 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 27454
16:25:55.846 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:25:55.846 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.173.61:39669
16:25:55.847 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
16:25:55.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.849069 26292 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:55.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:55.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/wal
16:25:55.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data
16:25:55.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:25:55.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:25:55.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:25:55.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:55.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:55.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/logs
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data/info.pb
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.61:39669
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.61
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46803
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.61
16:25:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:55.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:25:55.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:25:55.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:25:55.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:55.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:55.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:55.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:55.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:55.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:55.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:55.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33477/jwks.json
16:25:55.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:25:55.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:25:55.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.858232 27531 sys_catalog.cc:263] Verifying existing consensus state
16:25:55.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.872804 27531 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: Bootstrap starting.
16:25:55.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.916424 27531 log.cc:826] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: Log is configured to *not* fsync() on all Append() calls
16:25:55.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.936866 27531 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:25:55.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.937942 27531 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: Bootstrap complete.
16:25:55.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.967500 27531 raft_consensus.cc:359] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:55.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.970847 27531 raft_consensus.cc:740] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7724600eba9e4ebea1a0a1f3720ac88c, State: Initialized, Role: FOLLOWER
16:25:55.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.971907 27531 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:55.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.976675 27536 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:55.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.978058 27536 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: This master's current role is: FOLLOWER
16:25:55.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:55.982470 27531 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: configured and running, proceeding with master startup.
16:25:56.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.084110 27547 catalog_manager.cc:1269] Loaded cluster ID: 723802fca27f462db7efb81f8e18a050
16:25:56.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.084431 27547 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: loading cluster ID for follower catalog manager: success
16:25:56.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.092267 27547 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: acquiring CA information for follower catalog manager: success
16:25:56.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.097889 27547 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:25:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.059711 27454 master_runner.cc:186] Error getting master registration for 127.25.173.62:36591: Network error: Client connection negotiation failed: client connection to 127.25.173.62:36591: connect: Connection refused (error 111),
16:25:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.111135 27454 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:25:56.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.231710 27075 heartbeater.cc:344] Connected to a master server at 127.25.173.60:36837
16:25:56.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.241539 27496 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ac2de3172b094e988002346949fa14f1" instance_seqno: 1773764742734114) as {username='kudu', principal='kudu/127.25.173.3@KRBTEST.COM'} at 127.25.173.3:44073; Asking this server to re-register.
16:25:56.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.243809 27075 heartbeater.cc:461] Registering TS with master...
16:25:56.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.249063 27495 ts_manager.cc:194] Registered new tserver with Master: ac2de3172b094e988002346949fa14f1 (127.25.173.3:37197)
16:25:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.303544 26924 heartbeater.cc:344] Connected to a master server at 127.25.173.60:36837
16:25:56.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.306537 27496 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "212c40dfb16a4399b4215244f669ef7a" instance_seqno: 1773764739894621) as {username='kudu', principal='kudu/127.25.173.2@KRBTEST.COM'} at 127.25.173.2:33837; Asking this server to re-register.
16:25:56.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.310283 26924 heartbeater.cc:461] Registering TS with master...
16:25:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.313638 27496 ts_manager.cc:194] Registered new tserver with Master: 212c40dfb16a4399b4215244f669ef7a (127.25.173.2:43715)
16:25:56.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:56.397573 27534 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:25:56.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:56.398367 27534 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:25:56.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:56.399526 27534 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:25:56.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:56.443771 27534 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:25:56.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:56.444084 27534 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:25:56.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:56.444573 27534 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:25:56.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:56.444976 27534 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:25:56.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:56.445412 27534 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:25:56.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:56.445797 27534 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:25:56.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:56.446051 27534 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33477/jwks.json
16:25:56.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:56 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764756, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:56.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.463207 26660 init.cc:295] Successfully reacquired a new kerberos TGT
16:25:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.518759 27534 master_runner.cc:386] Master server non-default flags:
16:25:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data
16:25:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/wal
16:25:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:25:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.61
16:25:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.61:39669
16:25:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:25:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33477/jwks.json
16:25:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data/info.pb
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.61
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46803
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/logs
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:25:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:25:56.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.520866 27534 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:25:56.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.522349 27534 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:25:56.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:56.541999 27561 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.842881 26778 heartbeater.cc:344] Connected to a master server at 127.25.173.60:36837
16:25:56.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.847796 27496 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "3be9c2934e634e9db2674aa1fcd019af" instance_seqno: 1773764737282488) as {username='kudu', principal='kudu/127.25.173.1@KRBTEST.COM'} at 127.25.173.1:47649; Asking this server to re-register.
16:25:56.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.850310 26778 heartbeater.cc:461] Registering TS with master...
16:25:56.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:56.855001 27496 ts_manager.cc:194] Registered new tserver with Master: 3be9c2934e634e9db2674aa1fcd019af (127.25.173.1:42067)
16:25:57.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:57.186033 25641 debug-util.cc:398] Leaking SignalData structure 0x7b08000a4900 after lost signal to thread 25578
16:25:57.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:57.187454 25641 debug-util.cc:398] Leaking SignalData structure 0x7b080009e980 after lost signal to thread 25644
16:25:57.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:57.519572 27570 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:25:57.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:57.520951 27570 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:57.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:57.534905 27570 leader_election.cc:290] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 9bddceeba6654533b175280b4491784f (127.25.173.62:36591), 9555719ea49a4994a3973eb4452a45df (127.25.173.61:39669)
16:25:57.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:57.539594 27480 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.173.61:39669: connect: Connection refused (error 111)
16:25:57.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:57.560353 27480 leader_election.cc:336] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 9555719ea49a4994a3973eb4452a45df (127.25.173.61:39669): Network error: Client connection negotiation failed: client connection to 127.25.173.61:39669: connect: Connection refused (error 111)
16:25:57.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:57.564033 27482 leader_election.cc:336] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 9bddceeba6654533b175280b4491784f (127.25.173.62:36591): Network error: Client connection negotiation failed: client connection to 127.25.173.62:36591: connect: Connection refused (error 111)
16:25:57.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:57.574658 27482 leader_election.cc:304] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7724600eba9e4ebea1a0a1f3720ac88c; no voters: 9555719ea49a4994a3973eb4452a45df, 9bddceeba6654533b175280b4491784f
16:25:57.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:57.576311 27570 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
16:25:58.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:57.944484 27560 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 27534
16:25:58.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:58.535040 27560 kernel_stack_watchdog.cc:198] Thread 27534 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
16:25:58.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:25:58.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:25:58.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:25:58.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:25:58.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:56.543941 27562 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:58.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:58.540656 27534 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.999s user 0.700s sys 1.137s
16:25:58.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:58.541026 27534 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.000s user 0.700s sys 1.137s
16:25:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.549463 27534 server_base.cc:1060] running on GCE node
16:25:58.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:58.550274 27580 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.552136 27534 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:25:58.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.555912 27534 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:25:58.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.557688 27534 hybrid_clock.cc:648] HybridClock initialized: now 1773764758557612 us; error 83 us; skew 500 ppm
16:25:58.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.559229 27534 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:25:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:58 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764758, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:58.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.569370 27534 init.cc:377] Logged in from keytab as kudu/127.25.173.61@KRBTEST.COM (short username kudu)
16:25:58.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.579073 27534 webserver.cc:492] Webserver started at http://127.25.173.61:46803/ using document root <none> and password file <none>
16:25:58.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.580813 27534 fs_manager.cc:362] Metadata directory not provided
16:25:58.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.581408 27534 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:25:58.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.594332 27534 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.000s
16:25:58.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.601039 27586 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:25:58.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.602504 27534 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
16:25:58.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.602982 27534 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/wal
16:25:58.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9555719ea49a4994a3973eb4452a45df"
16:25:58.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:28 on dist-test-slave-jdhl"
16:25:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.605768 27534 fs_report.cc:389] FS layout report
16:25:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:25:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/wal
16:25:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/wal
16:25:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data/data
16:25:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:25:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:25:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:25:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:25:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:25:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:25:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:25:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:25:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:25:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:25:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:25:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.687644 27534 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:25:58.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.693663 27534 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:25:58.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.694221 27534 kserver.cc:163] Server-wide thread pool size limit: 3276
16:25:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.798830 27534 rpc_server.cc:307] RPC server started. Bound to: 127.25.173.61:39669
16:25:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.799778 27637 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.173.61:39669 every 8 connection(s)
16:25:58.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.802978 27534 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-1/data/info.pb
16:25:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.807535 26292 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 27534
16:25:58.808 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:25:58.808 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.173.62:36591
16:25:58.808 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.811764 26292 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/wal
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/logs
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data/info.pb
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.62:36591
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.62
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41617
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.62
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33477/jwks.json
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:25:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:25:58.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.820302 27638 sys_catalog.cc:263] Verifying existing consensus state
16:25:58.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.829231 27638 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: Bootstrap starting.
16:25:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.882746 27638 log.cc:826] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: Log is configured to *not* fsync() on all Append() calls
16:25:58.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.902133 27638 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:25:58.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.903131 27638 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: Bootstrap complete.
16:25:58.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.935670 27638 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:58.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.942302 27638 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9555719ea49a4994a3973eb4452a45df, State: Initialized, Role: FOLLOWER
16:25:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.943377 27638 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:58.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.955953 27644 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:25:58.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.959409 27644 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: This master's current role is: FOLLOWER
16:25:58.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.972117 27638 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: configured and running, proceeding with master startup.
16:25:58.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:58 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764756, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.1@KRBTEST.COM for kudu/127.25.173.61@KRBTEST.COM
16:25:58.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:58.997092 26780 heartbeater.cc:344] Connected to a master server at 127.25.173.61:39669
16:25:59.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.064749 27656 catalog_manager.cc:1269] Loaded cluster ID: 723802fca27f462db7efb81f8e18a050
16:25:59.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.065384 27656 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: loading cluster ID for follower catalog manager: success
16:25:59.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.049365 27534 master_runner.cc:186] Error getting master registration for 127.25.173.62:36591: Network error: Client connection negotiation failed: client connection to 127.25.173.62:36591: connect: Connection refused (error 111),
16:25:59.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.080724 27534 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:25:59.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.092259 27656 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: acquiring CA information for follower catalog manager: success
16:25:59.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.099867 27656 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:25:59.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:59.319013 27640 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:25:59.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:59.319867 27640 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:25:59.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.320097 27076 heartbeater.cc:344] Connected to a master server at 127.25.173.61:39669
16:25:59.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:59.320995 27640 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:25:59.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.328967 27603 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ac2de3172b094e988002346949fa14f1" instance_seqno: 1773764742734114) as {username='kudu', principal='kudu/127.25.173.3@KRBTEST.COM'} at 127.25.173.3:45043; Asking this server to re-register.
16:25:59.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.332280 27076 heartbeater.cc:461] Registering TS with master...
16:25:59.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.336751 27603 ts_manager.cc:194] Registered new tserver with Master: ac2de3172b094e988002346949fa14f1 (127.25.173.3:37197)
16:25:59.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:59.384802 27640 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:25:59.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:59.385126 27640 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:25:59.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:59.385601 27640 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:25:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:59.385986 27640 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:25:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:59.386313 27640 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:25:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:59.386631 27640 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:25:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:59.386911 27640 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33477/jwks.json
16:25:59.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.460466 26925 heartbeater.cc:344] Connected to a master server at 127.25.173.61:39669
16:25:59.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.460966 27640 master_runner.cc:386] Master server non-default flags:
16:25:59.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:25:59.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:41453
16:25:59.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:25:59.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:25:59.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data
16:25:59.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/wal
16:25:59.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:25:59.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:25:59.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:36591,127.25.173.61:39669,127.25.173.60:36837
16:25:59.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/krb5kdc/kudu.keytab
16:25:59.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.62
16:25:59.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:25:59.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:25:59.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.62:36591
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33477/jwks.json
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data/info.pb
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.62
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41617
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/logs
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:25:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:25:59.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:25:59.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:25:59.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:25:59.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.462716 27640 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:25:59.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.464262 27640 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:25:59.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.466183 27603 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "212c40dfb16a4399b4215244f669ef7a" instance_seqno: 1773764739894621) as {username='kudu', principal='kudu/127.25.173.2@KRBTEST.COM'} at 127.25.173.2:54783; Asking this server to re-register.
16:25:59.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.468763 26925 heartbeater.cc:461] Registering TS with master...
16:25:59.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.473101 27603 ts_manager.cc:194] Registered new tserver with Master: 212c40dfb16a4399b4215244f669ef7a (127.25.173.2:43715)
16:25:59.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:59.483531 27669 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:25:59.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:25:59 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764759, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:25:59.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.642818 26806 init.cc:295] Successfully reacquired a new kerberos TGT
16:25:59.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.879467 27675 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:25:59.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.880196 27675 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:25:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.925927 27675 leader_election.cc:290] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 9bddceeba6654533b175280b4491784f (127.25.173.62:36591), 9555719ea49a4994a3973eb4452a45df (127.25.173.61:39669)
16:25:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:59.928603 27482 leader_election.cc:336] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 9bddceeba6654533b175280b4491784f (127.25.173.62:36591): Network error: Client connection negotiation failed: client connection to 127.25.173.62:36591: connect: Connection refused (error 111)
16:25:59.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.962320 27613 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "9555719ea49a4994a3973eb4452a45df" is_pre_election: true
16:25:59.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.963373 27613 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7724600eba9e4ebea1a0a1f3720ac88c in term 2.
16:25:59.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.964852 27480 leader_election.cc:304] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7724600eba9e4ebea1a0a1f3720ac88c, 9555719ea49a4994a3973eb4452a45df; no voters: 9bddceeba6654533b175280b4491784f
16:25:59.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.966400 27675 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 2 FOLLOWER]: Leader pre-election won for term 3
16:25:59.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.967617 27675 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:25:59.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:25:59.968114 27675 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 2 FOLLOWER]: Advancing to term 3
16:26:00.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.004144 27603 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "3be9c2934e634e9db2674aa1fcd019af" instance_seqno: 1773764737282488) as {username='kudu', principal='kudu/127.25.173.1@KRBTEST.COM'} at 127.25.173.1:38497; Asking this server to re-register.
16:26:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.006493 26780 heartbeater.cc:461] Registering TS with master...
16:26:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.022162 27603 ts_manager.cc:194] Registered new tserver with Master: 3be9c2934e634e9db2674aa1fcd019af (127.25.173.1:42067)
16:26:00.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.059567 27675 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:26:00.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.083870 27613 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "9555719ea49a4994a3973eb4452a45df"
16:26:00.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.084705 27613 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 2 FOLLOWER]: Advancing to term 3
16:26:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:26:00.096514 27482 leader_election.cc:336] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 9bddceeba6654533b175280b4491784f (127.25.173.62:36591): Network error: Client connection negotiation failed: client connection to 127.25.173.62:36591: connect: Connection refused (error 111)
16:26:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.111696 27675 leader_election.cc:290] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 3 election: Requested vote from peers 9bddceeba6654533b175280b4491784f (127.25.173.62:36591), 9555719ea49a4994a3973eb4452a45df (127.25.173.61:39669)
16:26:00.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.124680 27613 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7724600eba9e4ebea1a0a1f3720ac88c in term 3.
16:26:00.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.157894 27480 leader_election.cc:304] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7724600eba9e4ebea1a0a1f3720ac88c, 9555719ea49a4994a3973eb4452a45df; no voters: 9bddceeba6654533b175280b4491784f
16:26:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.159945 27675 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 3 FOLLOWER]: Leader election won for term 3
16:26:00.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.171026 27675 raft_consensus.cc:697] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [term 3 LEADER]: Becoming Leader. State: Replica: 7724600eba9e4ebea1a0a1f3720ac88c, State: Running, Role: LEADER
16:26:00.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.180778 27675 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } }
16:26:00.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.254485 27680 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7724600eba9e4ebea1a0a1f3720ac88c. Latest consensus state: current_term: 3 leader_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:26:00.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.267294 27680 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: This master's current role is: LEADER
16:26:00.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.313738 27682 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
16:26:00.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.465301 27682 catalog_manager.cc:1494] Initializing Kudu cluster ID...
16:26:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.510475 27682 catalog_manager.cc:1269] Loaded cluster ID: 723802fca27f462db7efb81f8e18a050
16:26:00.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.512204 27682 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
16:26:00.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.534452 27682 catalog_manager.cc:1514] Loading token signing keys...
16:26:00.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.564517 27682 catalog_manager.cc:6055] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c: Loaded TSK: 0
16:26:00.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.582379 27682 catalog_manager.cc:1524] Initializing in-progress tserver states...
16:26:00.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:26:00.780105 27482 consensus_peers.cc:597] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c -> Peer 9bddceeba6654533b175280b4491784f (127.25.173.62:36591): Couldn't send request to peer 9bddceeba6654533b175280b4491784f. Status: Network error: Client connection negotiation failed: client connection to 127.25.173.62:36591: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
16:26:00.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.820119 27613 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [term 3 FOLLOWER]: Refusing update from remote peer 7724600eba9e4ebea1a0a1f3720ac88c: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
16:26:00.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.822677 27680 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [LEADER]: Connected to new peer: Peer: permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.001s
16:26:00.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.913362 27689 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7724600eba9e4ebea1a0a1f3720ac88c. Latest consensus state: current_term: 3 leader_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:26:00.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.914762 27689 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: This master's current role is: FOLLOWER
16:26:00.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.963621 27689 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:26:00.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.964362 27689 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9555719ea49a4994a3973eb4452a45df [sys.catalog]: This master's current role is: FOLLOWER
16:26:00.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.967144 27680 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bddceeba6654533b175280b4491784f" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 36591 } } peers { permanent_uuid: "9555719ea49a4994a3973eb4452a45df" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 39669 } } peers { permanent_uuid: "7724600eba9e4ebea1a0a1f3720ac88c" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 36837 } } }
16:26:00.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:00.973929 27680 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c [sys.catalog]: This master's current role is: LEADER
16:26:01.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:01.114223 27454 master_runner.cc:186] Error getting master registration for 127.25.173.62:36591: Network error: Client connection negotiation failed: client connection to 127.25.173.62:36591: connect: Connection refused (error 111),
16:26:01.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:01.114704 27454 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:26:01.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:26:01.732228 26276 debug-util.cc:398] Leaking SignalData structure 0x7b08000b40a0 after lost signal to thread 26151
16:26:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:26:01.733682 26276 debug-util.cc:398] Leaking SignalData structure 0x7b08000bacc0 after lost signal to thread 26279
16:26:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:26:01.873003 26136 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8240 after lost signal to thread 26011
16:26:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:26:01.874495 26136 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8800 after lost signal to thread 26139
16:26:01.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:01.885732 26778 heartbeater.cc:499] Master 127.25.173.60:36837 was elected leader, sending a full tablet report...
16:26:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.318845 27075 heartbeater.cc:499] Master 127.25.173.60:36837 was elected leader, sending a full tablet report...
16:26:02.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:26:00.884912 27668 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 27640
16:26:02.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:26:02.425890 27640 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.931s user 0.995s sys 1.426s
16:26:02.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:26:02.425995 27671 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2937 milliseconds
16:26:02.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:26:02.427208 27672 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:26:02.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:26:02.427713 27640 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.941s user 0.995s sys 1.426s
16:26:02.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.431519 27640 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:26:02.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:25:59.491333 27670 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:26:02.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.434296 27640 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:26:02.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.442366 27640 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:26:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.444919 27640 hybrid_clock.cc:648] HybridClock initialized: now 1773764762444841 us; error 85 us; skew 500 ppm
16:26:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.446708 27640 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:26:02.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:26:02 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764762, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:26:02.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.456710 26924 heartbeater.cc:499] Master 127.25.173.60:36837 was elected leader, sending a full tablet report...
16:26:02.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.462139 27640 init.cc:377] Logged in from keytab as kudu/127.25.173.62@KRBTEST.COM (short username kudu)
16:26:02.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:26:02 dist-test-slave-jdhl krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764762, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:26:02.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.469852 26957 init.cc:295] Successfully reacquired a new kerberos TGT
16:26:02.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.519750 27640 webserver.cc:492] Webserver started at http://127.25.173.62:41617/ using document root <none> and password file <none>
16:26:02.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.521286 27640 fs_manager.cc:362] Metadata directory not provided
16:26:02.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.521728 27640 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:26:02.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.548547 27640 fs_manager.cc:714] Time spent opening directory manager: real 0.022s user 0.015s sys 0.000s
16:26:02.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.562577 27708 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:26:02.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.564141 27640 fs_manager.cc:730] Time spent opening block manager: real 0.013s user 0.002s sys 0.007s
16:26:02.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.564618 27640 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/wal
16:26:02.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9bddceeba6654533b175280b4491784f"
16:26:02.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:25:25 on dist-test-slave-jdhl"
16:26:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.567615 27640 fs_report.cc:389] FS layout report
16:26:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:26:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/wal
16:26:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/wal
16:26:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data/data
16:26:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:26:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:26:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:26:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:26:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:26:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:26:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:26:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:26:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:26:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:26:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:26:02.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:26:02.679193 27482 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.173.62:36591: connect: Connection refused (error 111) [suppressed 7 similar messages]
16:26:02.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.713316 27640 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:26:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.730017 27640 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:26:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:02.731103 27640 kserver.cc:163] Server-wide thread pool size limit: 3276
16:26:03.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:26:03.062166 27482 consensus_peers.cc:597] T 00000000000000000000000000000000 P 7724600eba9e4ebea1a0a1f3720ac88c -> Peer 9bddceeba6654533b175280b4491784f (127.25.173.62:36591): Couldn't send request to peer 9bddceeba6654533b175280b4491784f. Status: Network error: Client connection negotiation failed: client connection to 127.25.173.62:36591: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
16:26:03.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:03.177446 27763 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.173.62:36591 every 8 connection(s)
16:26:03.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:03.178378 27640 rpc_server.cc:307] RPC server started. Bound to: 127.25.173.62:36591
16:26:03.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:03.183666 27640 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster7294855438186300349/master-0/data/info.pb
16:26:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:03.192678 26292 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 27640
16:26:03.194 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:26:03.195 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.173.60:36837(127.25.173.60:36837)] connecting to peer
16:26:03.221 [DEBUG - kudu-nio-3] (Connection.java:757) [peer master-127.25.173.60:36837(127.25.173.60:36837)] Successfully connected to peer
16:26:03.258 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.25.173.60:36837(127.25.173.60:36837)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
16:26:03.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:26:03.261598 27767 negotiation.cc:337] Failed RPC negotiation. Trace:
16:26:03.262 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.25.173.62:36591(127.25.173.62:36591)] disconnecting while in state TERMINATED
16:26:03.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:26:03.220683 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:43696 (local address 127.25.173.60:36837)
16:26:03.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:26:03.232823 (+ 12140us) server_negotiation.cc:207] Beginning negotiation
16:26:03.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:26:03.232863 (+ 40us) server_negotiation.cc:400] Waiting for connection header
16:26:03.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:26:03.250386 (+ 17523us) server_negotiation.cc:408] Connection header received
16:26:03.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:26:03.252639 (+ 2253us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
16:26:03.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:26:03.252678 (+ 39us) server_negotiation.cc:462] Received NEGOTIATE request from client
16:26:03.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:26:03.253117 (+ 439us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
16:26:03.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:26:03.253357 (+ 240us) server_negotiation.cc:227] Negotiated authn=SASL
16:26:03.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:26:03.259969 (+ 6612us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:43696: BlockingRecv error: recv got EOF from 127.0.0.1:43696 (error 108)
16:26:03.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":11373,"thread_start_us":11075,"threads_started":1}
16:26:03.264 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.25.173.61:39669(127.25.173.61:39669)] disconnecting while in state TERMINATED
16:26:03.264 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.25.173.60:36837(127.25.173.60:36837)] disconnecting while in state TERMINATED
16:26:03.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:03.283257 27764 sys_catalog.cc:263] Verifying existing consensus state
16:26:03.288 [DEBUG - kudu-nio-3] (Connection.java:239) [peer master-127.25.173.60:36837(127.25.173.60:36837)] handling channelInactive
16:26:03.290 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:26:03.303 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:26:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:03.312404 27764 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: Bootstrap starting.
16:26:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:03.347013 26292 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 26639
16:26:03.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:26:03 dist-test-slave-jdhl krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764762, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.3@KRBTEST.COM for kudu/127.25.173.62@KRBTEST.COM
16:26:03.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:03.475109 26292 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 26792
16:26:03.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:03.493052 27764 log.cc:826] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: Log is configured to *not* fsync() on all Append() calls
16:26:03.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:03.507967 27077 heartbeater.cc:344] Connected to a master server at 127.25.173.62:36591
16:26:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:03.546176 27764 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:26:03.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:03.548045 27764 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9bddceeba6654533b175280b4491784f: Bootstrap complete.
16:26:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:03.554019 26292 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 26938
16:26:03.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:03.630887 26292 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 27640
16:26:03.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:03.674808 26292 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 27534
16:26:03.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:26:03.677599 27778 negotiation.cc:337] Failed RPC negotiation. Trace:
16:26:03.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:26:03.657514 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.25.173.62:36591 (local address 127.0.0.1:52654)
16:26:03.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:26:03.666529 (+ 9015us) negotiation.cc:107] Waiting for socket to connect
16:26:03.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:26:03.666590 (+ 61us) client_negotiation.cc:174] Beginning negotiation
16:26:03.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:26:03.666927 (+ 337us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
16:26:03.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:26:03.676671 (+ 9744us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.25.173.62:36591: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
16:26:03.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":8291,"thread_start_us":11531,"threads_started":1}
16:26:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:03.737886 26292 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 27454
16:26:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-17T16:26:05Z chronyd exiting
16:26:06.297 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:26:06.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:06.299918 25564 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 25865
16:26:06.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:06.326850 25564 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 26009
16:26:06.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:06.357167 25564 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 26148
16:26:06.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:06.388439 25564 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 25577
16:26:06.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:06.422436 25564 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 25647
16:26:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:26:06.452948 25564 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 25722
16:26:07.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-17T16:26:07Z chronyd exiting
16:26:08.666 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more