03:06:43.559 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:06:43.559 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskoRAqye/test-tmp
03:06:43.560 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:06:43.560 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskoRAqye/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:06:43.562 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:06:43.897 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:06:43.897 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:06:43.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:43.900535 28691 mini_oidc.cc:150] Starting JWKS server
03:06:43.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:43.903761 28691 webserver.cc:492] Webserver started at http://127.0.0.1:43659/ using document root <none> and password file <none>
03:06:43.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:43.904225 28691 mini_oidc.cc:186] Starting OIDC Discovery server
03:06:43.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:43.905870 28691 webserver.cc:492] Webserver started at http://127.0.0.1:36061/ using document root <none> and password file <none>
03:06:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-20T03:06:43Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
03:06:43.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-20T03:06:43Z Disabled control of system clock
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:43.971903 28691 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-0/wal
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-0/data
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-0/logs
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-0/data/info.pb
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:06:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:06:43.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.4.254:40495
03:06:43.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.4.254
03:06:43.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:06:43.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.4.212:34923
03:06:43.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:06:43.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:06:43.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:06:43.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:06:43.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.4.254:40495,127.28.4.253:36601,127.28.4.252:43953
03:06:43.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:06:43.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43659/jwks.json with env {}
03:06:44.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:44.290377 28703 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:06:44.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:44.290985 28703 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:06:44.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:44.291483 28703 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:06:44.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:44.324038 28703 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:06:44.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:44.324424 28703 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:06:44.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:44.324649 28703 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:06:44.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:44.324847 28703 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:06:44.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:44.325044 28703 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:06:44.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:44.325193 28703 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43659/jwks.json
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:44.361934 28703 master_runner.cc:386] Master server non-default flags:
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.4.212:34923
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-0/data
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-0/wal
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.4.254:40495,127.28.4.253:36601,127.28.4.252:43953
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.4.254:40495
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43659/jwks.json
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-0/data/info.pb
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.4.254
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-0/logs
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:06:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:06:44.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:06:44.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:06:44.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:06:44.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:06:44.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:06:44.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:44.363319 28703 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:06:44.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:44.364965 28703 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:06:44.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:44.376107 28709 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:06:46.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:45.780431 28708 debug-util.cc:398] Leaking SignalData structure 0x7b080002edc0 after lost signal to thread 28703
03:06:46.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.219537 28703 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.844s user 0.678s sys 1.165s
03:06:46.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.220996 28703 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.845s user 0.678s sys 1.165s
03:06:46.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:44.376967 28710 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:06:46.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.222625 28712 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:06:46.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.225570 28711 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1844 milliseconds
03:06:46.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.225638 28703 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:06:46.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.226866 28703 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:06:46.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.229231 28703 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:06:46.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.230568 28703 hybrid_clock.cc:648] HybridClock initialized: now 1773976006230537 us; error 40 us; skew 500 ppm
03:06:46.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.231470 28703 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:06:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.237684 28703 webserver.cc:492] Webserver started at http://127.28.4.254:32833/ using document root <none> and password file <none>
03:06:46.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.238567 28703 fs_manager.cc:362] Metadata directory not provided
03:06:46.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.238838 28703 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:06:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.239262 28703 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:06:46.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.243656 28703 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-0/data/instance:
03:06:46.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5"
03:06:46.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:06:46 on dist-test-slave-sx75"
03:06:46.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.244793 28703 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-0/wal/instance:
03:06:46.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5"
03:06:46.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:06:46 on dist-test-slave-sx75"
03:06:46.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.251876 28703 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
03:06:46.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.257699 28720 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:06:46.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.258831 28703 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
03:06:46.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.259135 28703 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-0/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-0/wal
03:06:46.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5"
03:06:46.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:06:46 on dist-test-slave-sx75"
03:06:46.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.259505 28703 fs_report.cc:389] FS layout report
03:06:46.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:06:46.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-0/wal
03:06:46.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-0/wal
03:06:46.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-0/data/data
03:06:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:06:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:06:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:06:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:06:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:06:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:06:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:06:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:06:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:06:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:06:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:06:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.306182 28703 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:06:46.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.307746 28703 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:06:46.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.308246 28703 kserver.cc:163] Server-wide thread pool size limit: 3276
03:06:46.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.383280 28703 rpc_server.cc:307] RPC server started. Bound to: 127.28.4.254:40495
03:06:46.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.383355 28772 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.4.254:40495 every 8 connection(s)
03:06:46.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.386040 28703 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-0/data/info.pb
03:06:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.391752 28773 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:06:46.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.394568 28691 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 28703
03:06:46.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.395004 28691 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-0/wal/instance
03:06:46.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.398278 28691 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:06:46.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:06:46.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-1/wal
03:06:46.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-1/data
03:06:46.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:06:46.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:06:46.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:06:46.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:06:46.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:06:46.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:06:46.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-1/logs
03:06:46.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-1/data/info.pb
03:06:46.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:06:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:06:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:06:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:06:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:06:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:06:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:06:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:06:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:06:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:06:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:06:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:06:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.4.253:36601
03:06:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.4.253
03:06:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:06:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.4.212:34923
03:06:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:06:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:06:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:06:46.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:06:46.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.4.254:40495,127.28.4.253:36601,127.28.4.252:43953
03:06:46.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:06:46.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43659/jwks.json with env {}
03:06:46.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.403549 28773 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } has no permanent_uuid. Determining permanent_uuid...
03:06:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.431054 28773 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } has no permanent_uuid. Determining permanent_uuid...
03:06:46.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.433792 28723 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.4.253:36601: connect: Connection refused (error 111)
03:06:46.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.438496 28773 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.253:36601: Network error: Client connection negotiation failed: client connection to 127.28.4.253:36601: connect: Connection refused (error 111)
03:06:46.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.488070 28773 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } attempt: 1
03:06:46.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.493145 28773 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.253:36601: Network error: Client connection negotiation failed: client connection to 127.28.4.253:36601: connect: Connection refused (error 111)
03:06:46.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.561734 28773 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } attempt: 2
03:06:46.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.566315 28773 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.253:36601: Network error: Client connection negotiation failed: client connection to 127.28.4.253:36601: connect: Connection refused (error 111)
03:06:46.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.657760 28773 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } attempt: 3
03:06:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.662055 28773 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.253:36601: Network error: Client connection negotiation failed: client connection to 127.28.4.253:36601: connect: Connection refused (error 111)
03:06:46.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.713932 28775 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:06:46.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.714674 28775 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:06:46.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.715463 28775 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:06:46.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.747691 28775 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:06:46.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.748072 28775 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:06:46.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.748447 28775 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:06:46.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.748775 28775 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:06:46.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.749094 28775 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:06:46.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.749346 28775 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43659/jwks.json
03:06:46.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.786284 28775 master_runner.cc:386] Master server non-default flags:
03:06:46.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:06:46.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.4.212:34923
03:06:46.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:06:46.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:06:46.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-1/data
03:06:46.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-1/wal
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.4.254:40495,127.28.4.253:36601,127.28.4.252:43953
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.4.253:36601
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43659/jwks.json
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-1/data/info.pb
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.4.253
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-1/logs
03:06:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:06:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:06:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:06:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:06:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:06:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:06:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:06:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:06:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:06:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:06:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.788036 28775 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:06:46.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.789786 28775 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:06:46.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.801364 28784 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:06:46.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:46.805495 28773 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } attempt: 4
03:06:46.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.812494 28773 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.253:36601: Network error: Client connection negotiation failed: client connection to 127.28.4.253:36601: connect: Connection refused (error 111)
03:06:47.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:47.112056 28773 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } attempt: 5
03:06:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:47.120599 28773 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.253:36601: Network error: Client connection negotiation failed: client connection to 127.28.4.253:36601: connect: Connection refused (error 111)
03:06:47.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:47.668170 28773 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } attempt: 6
03:06:47.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:47.694818 28773 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.253:36601: Network error: Client connection negotiation failed: client connection to 127.28.4.253:36601: connect: Connection refused (error 111)
03:06:48.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.206115 28783 debug-util.cc:398] Leaking SignalData structure 0x7b080002edc0 after lost signal to thread 28775
03:06:48.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.303965 28775 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.503s user 0.518s sys 0.984s
03:06:48.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.304456 28775 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.504s user 0.518s sys 0.984s
03:06:48.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:46.801539 28785 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:06:48.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.306039 28786 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1502 milliseconds
03:06:48.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.306452 28787 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:06:48.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.306641 28775 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:06:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.310474 28775 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:06:48.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.313051 28775 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:06:48.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.314440 28775 hybrid_clock.cc:648] HybridClock initialized: now 1773976008314398 us; error 47 us; skew 500 ppm
03:06:48.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.315354 28775 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:06:48.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.321838 28775 webserver.cc:492] Webserver started at http://127.28.4.253:34967/ using document root <none> and password file <none>
03:06:48.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.322821 28775 fs_manager.cc:362] Metadata directory not provided
03:06:48.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.323029 28775 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:06:48.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.323426 28775 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:06:48.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.327999 28775 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-1/data/instance:
03:06:48.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fc41d651c13646568e55d291615bf6a5"
03:06:48.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:06:48 on dist-test-slave-sx75"
03:06:48.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.329200 28775 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-1/wal/instance:
03:06:48.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fc41d651c13646568e55d291615bf6a5"
03:06:48.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:06:48 on dist-test-slave-sx75"
03:06:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.336692 28775 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
03:06:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.342629 28796 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:06:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.343976 28775 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
03:06:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.344338 28775 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-1/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-1/wal
03:06:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fc41d651c13646568e55d291615bf6a5"
03:06:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:06:48 on dist-test-slave-sx75"
03:06:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.344655 28775 fs_report.cc:389] FS layout report
03:06:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:06:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-1/wal
03:06:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-1/wal
03:06:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-1/data/data
03:06:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:06:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:06:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:06:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:06:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:06:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:06:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:06:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:06:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:06:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:06:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:06:48.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.421551 28775 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:06:48.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.423094 28775 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:06:48.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.423544 28775 kserver.cc:163] Server-wide thread pool size limit: 3276
03:06:48.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.496029 28775 rpc_server.cc:307] RPC server started. Bound to: 127.28.4.253:36601
03:06:48.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.496100 28848 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.4.253:36601 every 8 connection(s)
03:06:48.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.498881 28775 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-1/data/info.pb
03:06:48.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.500779 28691 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 28775
03:06:48.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.501286 28691 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-1/wal/instance
03:06:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.504539 28691 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-2/wal
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-2/data
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-2/logs
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-2/data/info.pb
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.4.252:43953
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.4.252
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.4.212:34923
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.4.254:40495,127.28.4.253:36601,127.28.4.252:43953
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:06:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43659/jwks.json with env {}
03:06:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.506907 28849 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:06:48.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.520972 28849 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } has no permanent_uuid. Determining permanent_uuid...
03:06:48.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.547106 28849 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } has no permanent_uuid. Determining permanent_uuid...
03:06:48.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.562646 28849 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } has no permanent_uuid. Determining permanent_uuid...
03:06:48.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.565064 28799 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.4.252:43953: connect: Connection refused (error 111)
03:06:48.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.569183 28849 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.252:43953: Network error: Client connection negotiation failed: client connection to 127.28.4.252:43953: connect: Connection refused (error 111)
03:06:48.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.618796 28849 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } attempt: 1
03:06:48.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.623165 28849 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.252:43953: Network error: Client connection negotiation failed: client connection to 127.28.4.252:43953: connect: Connection refused (error 111)
03:06:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.691601 28849 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } attempt: 2
03:06:48.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.695928 28849 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.252:43953: Network error: Client connection negotiation failed: client connection to 127.28.4.252:43953: connect: Connection refused (error 111)
03:06:48.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.755398 28773 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } attempt: 7
03:06:48.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.768008 28773 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } has no permanent_uuid. Determining permanent_uuid...
03:06:48.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.774106 28773 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.252:43953: Network error: Client connection negotiation failed: client connection to 127.28.4.252:43953: connect: Connection refused (error 111)
03:06:48.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.787303 28849 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } attempt: 3
03:06:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.791468 28849 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.252:43953: Network error: Client connection negotiation failed: client connection to 127.28.4.252:43953: connect: Connection refused (error 111)
03:06:48.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.831773 28851 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:06:48.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.832479 28851 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:06:48.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.832562 28773 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } attempt: 1
03:06:48.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.833148 28851 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:06:48.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.836858 28773 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.252:43953: Network error: Client connection negotiation failed: client connection to 127.28.4.252:43953: connect: Connection refused (error 111)
03:06:48.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.867753 28851 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:06:48.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.868026 28851 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:06:48.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.868278 28851 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:06:48.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.868484 28851 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:06:48.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.868675 28851 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:06:48.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.868822 28851 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43659/jwks.json
03:06:48.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.904505 28851 master_runner.cc:386] Master server non-default flags:
03:06:48.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.4.212:34923
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-2/data
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-2/wal
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.4.254:40495,127.28.4.253:36601,127.28.4.252:43953
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.4.252:43953
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43659/jwks.json
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-2/data/info.pb
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.4.252
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-2/logs
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:06:48.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:06:48.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.905856 28851 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:06:48.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.907750 28851 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:06:48.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.918256 28773 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } attempt: 2
03:06:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.921386 28864 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:06:48.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.924948 28773 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.252:43953: Network error: Client connection negotiation failed: client connection to 127.28.4.252:43953: connect: Connection refused (error 111)
03:06:48.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:48.934828 28849 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } attempt: 4
03:06:48.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.942322 28849 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.252:43953: Network error: Client connection negotiation failed: client connection to 127.28.4.252:43953: connect: Connection refused (error 111)
03:06:49.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:49.010511 28773 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } attempt: 3
03:06:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:49.018184 28773 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.252:43953: Network error: Client connection negotiation failed: client connection to 127.28.4.252:43953: connect: Connection refused (error 111)
03:06:49.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:49.158768 28773 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } attempt: 4
03:06:49.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:49.166316 28773 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.252:43953: Network error: Client connection negotiation failed: client connection to 127.28.4.252:43953: connect: Connection refused (error 111)
03:06:49.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:49.242108 28849 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } attempt: 5
03:06:49.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:49.249856 28849 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.252:43953: Network error: Client connection negotiation failed: client connection to 127.28.4.252:43953: connect: Connection refused (error 111)
03:06:49.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:49.449875 28773 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } attempt: 5
03:06:49.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:49.458791 28773 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.252:43953: Network error: Client connection negotiation failed: client connection to 127.28.4.252:43953: connect: Connection refused (error 111)
03:06:49.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:49.797488 28849 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } attempt: 6
03:06:49.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:49.825595 28849 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.252:43953: Network error: Client connection negotiation failed: client connection to 127.28.4.252:43953: connect: Connection refused (error 111)
03:06:50.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.011411 28773 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } attempt: 6
03:06:50.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:50.042593 28773 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.4.252:43953: Network error: Client connection negotiation failed: client connection to 127.28.4.252:43953: connect: Connection refused (error 111)
03:06:50.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:48.924958 28865 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:06:50.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:50.290222 28867 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:06:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:50.293285 28866 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1362 milliseconds
03:06:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:50.293442 28851 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.370s user 0.431s sys 0.931s
03:06:50.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:50.293788 28851 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.370s user 0.431s sys 0.931s
03:06:50.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.294111 28851 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:06:50.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.295431 28851 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:06:50.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.298063 28851 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:06:50.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.299466 28851 hybrid_clock.cc:648] HybridClock initialized: now 1773976010299416 us; error 60 us; skew 500 ppm
03:06:50.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.300336 28851 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:06:50.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.308588 28851 webserver.cc:492] Webserver started at http://127.28.4.252:34417/ using document root <none> and password file <none>
03:06:50.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.309595 28851 fs_manager.cc:362] Metadata directory not provided
03:06:50.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.309847 28851 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:06:50.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.310360 28851 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:06:50.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.314817 28851 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-2/data/instance:
03:06:50.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd"
03:06:50.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:06:50 on dist-test-slave-sx75"
03:06:50.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.316012 28851 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-2/wal/instance:
03:06:50.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd"
03:06:50.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:06:50 on dist-test-slave-sx75"
03:06:50.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.324512 28851 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.003s
03:06:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.330930 28878 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:06:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.332216 28851 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
03:06:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.332619 28851 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-2/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-2/wal
03:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd"
03:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:06:50 on dist-test-slave-sx75"
03:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.332975 28851 fs_report.cc:389] FS layout report
03:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-2/wal
03:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-2/wal
03:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-2/data/data
03:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:06:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.413941 28851 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:06:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.415488 28851 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:06:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.415901 28851 kserver.cc:163] Server-wide thread pool size limit: 3276
03:06:50.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.487666 28851 rpc_server.cc:307] RPC server started. Bound to: 127.28.4.252:43953
03:06:50.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.487757 28929 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.4.252:43953 every 8 connection(s)
03:06:50.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.490339 28851 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-2/data/info.pb
03:06:50.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.495473 28691 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 28851
03:06:50.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.495908 28691 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/master-2/wal/instance
03:06:50.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.497534 28930 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:06:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.513403 28930 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } has no permanent_uuid. Determining permanent_uuid...
03:06:50.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.540479 28930 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } has no permanent_uuid. Determining permanent_uuid...
03:06:50.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.551803 28930 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } has no permanent_uuid. Determining permanent_uuid...
03:06:50.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.570317 28930 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd: Bootstrap starting.
03:06:50.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.577442 28930 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd: Neither blocks nor log segments found. Creating new log.
03:06:50.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.579301 28930 log.cc:826] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd: Log is configured to *not* fsync() on all Append() calls
03:06:50.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.584513 28930 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd: No bootstrap required, opened a new log
03:06:50.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.603276 28930 raft_consensus.cc:359] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } }
03:06:50.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.604149 28930 raft_consensus.cc:385] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:06:50.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.604453 28930 raft_consensus.cc:740] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d11d37c9bdaa48278f15e0a64ae8b8cd, State: Initialized, Role: FOLLOWER
03:06:50.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.605406 28930 consensus_queue.cc:260] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } }
03:06:50.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.608249 28939 sys_catalog.cc:455] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } } }
03:06:50.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.608945 28939 sys_catalog.cc:458] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [sys.catalog]: This master's current role is: FOLLOWER
03:06:50.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.609581 28930 sys_catalog.cc:565] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [sys.catalog]: configured and running, proceeding with master startup.
03:06:50.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:50.634475 28950 catalog_manager.cc:1568] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:06:50.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:50.634873 28950 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:06:50.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.629550 28851 master_runner.cc:186] Error getting master registration for 127.28.4.254:40495: OK, instance_id { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" instance_seqno: 1773976006363147 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:06:50.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.660542 28851 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:06:50.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.668936 28939 raft_consensus.cc:493] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:06:50.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.669451 28939 raft_consensus.cc:515] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } }
03:06:50.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:50.670882 28884 tablet.cc:2379] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:06:50.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.671927 28939 leader_election.cc:290] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 14cd0a94c8754fda8a4f1aacce6cdcb5 (127.28.4.254:40495), fc41d651c13646568e55d291615bf6a5 (127.28.4.253:36601)
03:06:50.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.886101 28849 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } attempt: 7
03:06:50.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.908615 28849 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5: Bootstrap starting.
03:06:50.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.915463 28849 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5: Neither blocks nor log segments found. Creating new log.
03:06:50.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.920912 28849 log.cc:826] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5: Log is configured to *not* fsync() on all Append() calls
03:06:50.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.926395 28849 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5: No bootstrap required, opened a new log
03:06:50.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.953990 28849 raft_consensus.cc:359] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } }
03:06:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.955158 28849 raft_consensus.cc:385] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:06:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.955603 28849 raft_consensus.cc:740] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fc41d651c13646568e55d291615bf6a5, State: Initialized, Role: FOLLOWER
03:06:50.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.956544 28849 consensus_queue.cc:260] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } }
03:06:50.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.964996 28954 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } } }
03:06:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.965837 28954 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [sys.catalog]: This master's current role is: FOLLOWER
03:06:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.967881 28954 raft_consensus.cc:493] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:06:50.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.968883 28849 sys_catalog.cc:565] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [sys.catalog]: configured and running, proceeding with master startup.
03:06:50.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.969031 28824 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fc41d651c13646568e55d291615bf6a5" is_pre_election: true
03:06:50.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.969805 28954 raft_consensus.cc:515] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } }
03:06:50.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.971165 28824 raft_consensus.cc:2468] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d11d37c9bdaa48278f15e0a64ae8b8cd in term 0.
03:06:50.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.979071 28880 leader_election.cc:304] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d11d37c9bdaa48278f15e0a64ae8b8cd, fc41d651c13646568e55d291615bf6a5; no voters:
03:06:50.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.980082 28939 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [term 0 FOLLOWER]: Leader pre-election won for term 1
03:06:50.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.980435 28939 raft_consensus.cc:493] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:06:50.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.980759 28939 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [term 0 FOLLOWER]: Advancing to term 1
03:06:50.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.991015 28939 raft_consensus.cc:515] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } }
03:06:50.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.998831 28905 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fc41d651c13646568e55d291615bf6a5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" is_pre_election: true
03:06:51.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:50.999791 28905 raft_consensus.cc:2393] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate fc41d651c13646568e55d291615bf6a5 in current term 1: Already voted for candidate d11d37c9bdaa48278f15e0a64ae8b8cd in this term.
03:06:51.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.001592 28939 leader_election.cc:290] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [CANDIDATE]: Term 1 election: Requested vote from peers 14cd0a94c8754fda8a4f1aacce6cdcb5 (127.28.4.254:40495), fc41d651c13646568e55d291615bf6a5 (127.28.4.253:36601)
03:06:51.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.001897 28824 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fc41d651c13646568e55d291615bf6a5"
03:06:51.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.001791 28954 leader_election.cc:290] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 14cd0a94c8754fda8a4f1aacce6cdcb5 (127.28.4.254:40495), d11d37c9bdaa48278f15e0a64ae8b8cd (127.28.4.252:43953)
03:06:51.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.003863 28824 raft_consensus.cc:3060] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [term 0 FOLLOWER]: Advancing to term 1
03:06:51.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.038003 28824 raft_consensus.cc:2468] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d11d37c9bdaa48278f15e0a64ae8b8cd in term 1.
03:06:51.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.040948 28880 leader_election.cc:304] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d11d37c9bdaa48278f15e0a64ae8b8cd, fc41d651c13646568e55d291615bf6a5; no voters:
03:06:51.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.041960 28939 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [term 1 FOLLOWER]: Leader election won for term 1
03:06:51.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.053298 28939 raft_consensus.cc:697] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [term 1 LEADER]: Becoming Leader. State: Replica: d11d37c9bdaa48278f15e0a64ae8b8cd, State: Running, Role: LEADER
03:06:51.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.054294 28939 consensus_queue.cc:237] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } }
03:06:51.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.076188 28773 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } attempt: 7
03:06:51.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.098968 28962 sys_catalog.cc:455] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [sys.catalog]: SysCatalogTable state changed. Reason: New leader d11d37c9bdaa48278f15e0a64ae8b8cd. Latest consensus state: current_term: 1 leader_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } } }
03:06:51.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.099740 28962 sys_catalog.cc:458] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [sys.catalog]: This master's current role is: LEADER
03:06:51.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.102828 28970 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:06:51.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:51.111197 28967 catalog_manager.cc:1568] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:06:51.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.111204 28773 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5: Bootstrap starting.
03:06:51.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:51.111565 28967 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:06:51.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.120986 28970 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:06:51.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.124641 28773 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5: Neither blocks nor log segments found. Creating new log.
03:06:51.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.129766 28773 log.cc:826] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5: Log is configured to *not* fsync() on all Append() calls
03:06:51.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.144280 28773 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5: No bootstrap required, opened a new log
03:06:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.087288 28775 master_runner.cc:186] Error getting master registration for 127.28.4.254:40495: OK, instance_id { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" instance_seqno: 1773976006363147 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:06:51.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.164048 28775 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:06:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.170578 28824 raft_consensus.cc:1275] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [term 1 FOLLOWER]: Refusing update from remote peer d11d37c9bdaa48278f15e0a64ae8b8cd: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:06:51.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.172839 28939 consensus_queue.cc:1048] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:06:51.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:51.181624 28802 tablet.cc:2379] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:06:51.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.184322 28773 raft_consensus.cc:359] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } }
03:06:51.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.186676 28773 raft_consensus.cc:385] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:06:51.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.187245 28773 raft_consensus.cc:740] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 14cd0a94c8754fda8a4f1aacce6cdcb5, State: Initialized, Role: FOLLOWER
03:06:51.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.188517 28773 consensus_queue.cc:260] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } }
03:06:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.204921 28973 sys_catalog.cc:455] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } } }
03:06:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.206362 28973 sys_catalog.cc:458] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5 [sys.catalog]: This master's current role is: FOLLOWER
03:06:51.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.208357 28773 sys_catalog.cc:565] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5 [sys.catalog]: configured and running, proceeding with master startup.
03:06:51.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.208544 28747 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5"
03:06:51.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.209398 28747 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5 [term 0 FOLLOWER]: Advancing to term 1
03:06:51.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.214861 28746 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fc41d651c13646568e55d291615bf6a5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" is_pre_election: true
03:06:51.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.217315 28800 leader_election.cc:304] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fc41d651c13646568e55d291615bf6a5; no voters: 14cd0a94c8754fda8a4f1aacce6cdcb5, d11d37c9bdaa48278f15e0a64ae8b8cd
03:06:51.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.219465 28748 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" is_pre_election: true
03:06:51.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.228397 28747 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d11d37c9bdaa48278f15e0a64ae8b8cd in term 1.
03:06:51.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.230911 28745 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5 [term 1 FOLLOWER]: Refusing update from remote peer d11d37c9bdaa48278f15e0a64ae8b8cd: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:06:51.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.239681 28939 consensus_queue.cc:1048] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [LEADER]: Connected to new peer: Peer: permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
03:06:51.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.267448 28975 raft_consensus.cc:2749] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:06:51.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.268780 28973 sys_catalog.cc:455] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d11d37c9bdaa48278f15e0a64ae8b8cd. Latest consensus state: current_term: 1 leader_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } } }
03:06:51.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.269554 28973 sys_catalog.cc:458] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5 [sys.catalog]: This master's current role is: FOLLOWER
03:06:51.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.272109 28954 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d11d37c9bdaa48278f15e0a64ae8b8cd. Latest consensus state: current_term: 1 leader_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } } }
03:06:51.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.272814 28954 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [sys.catalog]: This master's current role is: FOLLOWER
03:06:51.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.278206 28972 mvcc.cc:204] Tried to move back new op lower bound from 7266205741737140224 to 7266205741288775680. Current Snapshot: MvccSnapshot[applied={T|T < 7266205741737140224}]
03:06:51.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.287505 28970 catalog_manager.cc:1357] Generated new cluster ID: 8c3bf4608edf4f82b5d579f54f5fb83b
03:06:51.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.287513 28981 mvcc.cc:204] Tried to move back new op lower bound from 7266205741737140224 to 7266205741288775680. Current Snapshot: MvccSnapshot[applied={T|T < 7266205741737140224}]
03:06:51.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.287864 28970 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:06:51.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.289309 28962 sys_catalog.cc:455] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } } }
03:06:51.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.290076 28962 sys_catalog.cc:458] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [sys.catalog]: This master's current role is: LEADER
03:06:51.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.291031 28939 sys_catalog.cc:455] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } } }
03:06:51.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.291826 28939 sys_catalog.cc:458] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd [sys.catalog]: This master's current role is: LEADER
03:06:51.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.299949 28973 sys_catalog.cc:455] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } } }
03:06:51.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.300637 28973 sys_catalog.cc:458] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5 [sys.catalog]: This master's current role is: FOLLOWER
03:06:51.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.309437 28954 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "14cd0a94c8754fda8a4f1aacce6cdcb5" member_type: VOTER last_known_addr { host: "127.28.4.254" port: 40495 } } peers { permanent_uuid: "fc41d651c13646568e55d291615bf6a5" member_type: VOTER last_known_addr { host: "127.28.4.253" port: 36601 } } peers { permanent_uuid: "d11d37c9bdaa48278f15e0a64ae8b8cd" member_type: VOTER last_known_addr { host: "127.28.4.252" port: 43953 } } }
03:06:51.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.310194 28954 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5 [sys.catalog]: This master's current role is: FOLLOWER
03:06:51.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.357455 28970 catalog_manager.cc:1380] Generated new certificate authority record
03:06:51.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.359373 28970 catalog_manager.cc:1514] Loading token signing keys...
03:06:51.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.384949 28992 catalog_manager.cc:1269] Loaded cluster ID: 8c3bf4608edf4f82b5d579f54f5fb83b
03:06:51.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.394812 28992 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5: loading cluster ID for follower catalog manager: success
03:06:51.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.402668 28992 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5: acquiring CA information for follower catalog manager: success
03:06:51.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:51.407045 28992 catalog_manager.cc:883] Not found: no TSK found in the system table: failed to prepare follower catalog manager, will retry
03:06:51.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.427074 28970 catalog_manager.cc:6044] T 00000000000000000000000000000000 P d11d37c9bdaa48278f15e0a64ae8b8cd: Generated new TSK 0
03:06:51.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.428205 28970 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:51.474644 28691 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-0/wal
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-0/data
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-0/logs
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-0/data/info.pb
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.4.193:0
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.4.193
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.4.193
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.4.254:40495,127.28.4.253:36601,127.28.4.252:43953
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.4.212:34923
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:06:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:06:51.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:51.982002 28994 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:06:51.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:51.982944 28994 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:06:51.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:51.983868 28994 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:06:52.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:52.043852 28994 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:06:52.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:52.045475 28994 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.4.193
03:06:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:52.110759 28994 tablet_server_runner.cc:78] Tablet server non-default flags:
03:06:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:06:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.4.212:34923
03:06:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:06:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:06:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-0/data
03:06:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-0/wal
03:06:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:06:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:06:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.4.193:0
03:06:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:06:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:06:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:06:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-0/data/info.pb
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.4.193
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.4.254:40495,127.28.4.253:36601,127.28.4.252:43953
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.4.193
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-0/logs
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:06:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:06:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:52.114068 28994 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:06:52.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:52.116454 28994 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:06:52.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:52.117290 28967 catalog_manager.cc:1269] Loaded cluster ID: 8c3bf4608edf4f82b5d579f54f5fb83b
03:06:52.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:52.117681 28967 catalog_manager.cc:1562] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5: loading cluster ID for follower catalog manager: success
03:06:52.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:52.125238 28967 catalog_manager.cc:1584] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5: acquiring CA information for follower catalog manager: success
03:06:52.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:52.131829 28967 catalog_manager.cc:1612] T 00000000000000000000000000000000 P fc41d651c13646568e55d291615bf6a5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:06:52.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:52.167529 29000 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:06:52.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:52.427143 28992 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 14cd0a94c8754fda8a4f1aacce6cdcb5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:06:54.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:53.547464 28999 debug-util.cc:398] Leaking SignalData structure 0x7b080002eda0 after lost signal to thread 28994
03:06:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:54.884694 28994 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.721s user 0.935s sys 1.538s
03:06:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:54.885409 28994 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.722s user 0.935s sys 1.538s
03:06:54.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:52.170688 29001 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:06:54.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.895694 28994 server_base.cc:1061] running on GCE node
03:06:54.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:54.897845 29007 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:06:54.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.899142 28994 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:06:54.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.901793 28994 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:06:54.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.903239 28994 hybrid_clock.cc:648] HybridClock initialized: now 1773976014903143 us; error 98 us; skew 500 ppm
03:06:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.904181 28994 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:06:54.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.911530 28994 webserver.cc:492] Webserver started at http://127.28.4.193:43955/ using document root <none> and password file <none>
03:06:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.912703 28994 fs_manager.cc:362] Metadata directory not provided
03:06:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.913058 28994 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:06:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.913725 28994 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:06:54.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.920580 28994 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-0/data/instance:
03:06:54.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c61cbc0f65e438085ce5a4ad12c6951"
03:06:54.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:06:54 on dist-test-slave-sx75"
03:06:54.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.921756 28994 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-0/wal/instance:
03:06:54.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c61cbc0f65e438085ce5a4ad12c6951"
03:06:54.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:06:54 on dist-test-slave-sx75"
03:06:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.929958 28994 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.009s sys 0.001s
03:06:54.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.936627 29012 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:06:54.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.938771 28994 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
03:06:54.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.939241 28994 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-0/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-0/wal
03:06:54.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c61cbc0f65e438085ce5a4ad12c6951"
03:06:54.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:06:54 on dist-test-slave-sx75"
03:06:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.939622 28994 fs_report.cc:389] FS layout report
03:06:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:06:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-0/wal
03:06:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-0/wal
03:06:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-0/data/data
03:06:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:06:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:06:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:06:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:06:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:06:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:06:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:06:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:06:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:06:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:06:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:06:54.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.994235 28994 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:06:54.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.995889 28994 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:06:54.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.996410 28994 kserver.cc:163] Server-wide thread pool size limit: 3276
03:06:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:54.999684 28994 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:06:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.004840 28994 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:06:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.005079 28994 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:06:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.005414 28994 ts_tablet_manager.cc:616] Registered 0 tablets
03:06:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.005611 28994 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:06:55.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.231426 28994 rpc_server.cc:307] RPC server started. Bound to: 127.28.4.193:40031
03:06:55.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.231575 29128 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.4.193:40031 every 8 connection(s)
03:06:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.234100 28994 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-0/data/info.pb
03:06:55.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.237279 28691 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 28994
03:06:55.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.237926 28691 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-0/wal/instance
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.249557 28691 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-1/wal
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-1/data
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-1/logs
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-1/data/info.pb
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.4.194:0
03:06:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.4.194
03:06:55.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.4.194
03:06:55.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:06:55.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.4.254:40495,127.28.4.253:36601,127.28.4.252:43953
03:06:55.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.4.212:34923
03:06:55.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:06:55.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:06:55.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:06:55.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.273540 29129 heartbeater.cc:344] Connected to a master server at 127.28.4.252:43953
03:06:55.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.275660 29129 heartbeater.cc:461] Registering TS with master...
03:06:55.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.277100 29129 heartbeater.cc:507] Master 127.28.4.252:43953 requested a full tablet report, sending...
03:06:55.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.280890 28895 ts_manager.cc:194] Registered new tserver with Master: 9c61cbc0f65e438085ce5a4ad12c6951 (127.28.4.193:40031)
03:06:55.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.283556 28895 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.4.193:55533
03:06:55.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.290995 29130 heartbeater.cc:344] Connected to a master server at 127.28.4.253:36601
03:06:55.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.291380 29130 heartbeater.cc:461] Registering TS with master...
03:06:55.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.292058 29130 heartbeater.cc:507] Master 127.28.4.253:36601 requested a full tablet report, sending...
03:06:55.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.293015 29134 heartbeater.cc:344] Connected to a master server at 127.28.4.254:40495
03:06:55.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.293375 29134 heartbeater.cc:461] Registering TS with master...
03:06:55.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.295563 28814 ts_manager.cc:194] Registered new tserver with Master: 9c61cbc0f65e438085ce5a4ad12c6951 (127.28.4.193:40031)
03:06:55.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.296654 29134 heartbeater.cc:507] Master 127.28.4.254:40495 requested a full tablet report, sending...
03:06:55.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.300446 28738 ts_manager.cc:194] Registered new tserver with Master: 9c61cbc0f65e438085ce5a4ad12c6951 (127.28.4.193:40031)
03:06:55.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:55.592242 29136 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:06:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:55.592806 29136 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:06:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:55.593372 29136 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:06:55.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:55.626969 29136 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:06:55.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:55.627890 29136 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.4.194
03:06:55.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.664492 29136 tablet_server_runner.cc:78] Tablet server non-default flags:
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.4.212:34923
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-1/data
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-1/wal
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.4.194:0
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-1/data/info.pb
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.4.194
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.4.254:40495,127.28.4.253:36601,127.28.4.252:43953
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.4.194
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-1/logs
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:06:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:06:55.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.665931 29136 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:06:55.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:55.667881 29136 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:06:55.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:55.682224 29146 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:06:56.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:56.287813 29129 heartbeater.cc:499] Master 127.28.4.252:43953 was elected leader, sending a full tablet report...
03:06:57.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:55.683662 29147 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:06:57.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:57.089327 29144 debug-util.cc:398] Leaking SignalData structure 0x7b080002eda0 after lost signal to thread 29136
03:06:57.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:57.398500 29136 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.716s user 0.608s sys 1.041s
03:06:57.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:57.400648 29136 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.718s user 0.608s sys 1.041s
03:06:57.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:57.401526 29150 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:06:57.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:57.404853 29148 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1715 milliseconds
03:06:57.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.404953 29136 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:06:57.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.406441 29136 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:06:57.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.408921 29136 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:06:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.410352 29136 hybrid_clock.cc:648] HybridClock initialized: now 1773976017410303 us; error 35 us; skew 500 ppm
03:06:57.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.411383 29136 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:06:57.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.418681 29136 webserver.cc:492] Webserver started at http://127.28.4.194:42499/ using document root <none> and password file <none>
03:06:57.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.419871 29136 fs_manager.cc:362] Metadata directory not provided
03:06:57.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.420133 29136 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:06:57.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.420661 29136 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:06:57.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.426730 29136 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-1/data/instance:
03:06:57.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c52eee18c93483fa50cba6316720ce8"
03:06:57.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:06:57 on dist-test-slave-sx75"
03:06:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.427887 29136 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-1/wal/instance:
03:06:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c52eee18c93483fa50cba6316720ce8"
03:06:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:06:57 on dist-test-slave-sx75"
03:06:57.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.434841 29136 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.002s
03:06:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.440881 29158 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:06:57.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.442073 29136 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
03:06:57.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.442358 29136 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-1/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-1/wal
03:06:57.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c52eee18c93483fa50cba6316720ce8"
03:06:57.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:06:57 on dist-test-slave-sx75"
03:06:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.442651 29136 fs_report.cc:389] FS layout report
03:06:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:06:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-1/wal
03:06:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-1/wal
03:06:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-1/data/data
03:06:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:06:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:06:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:06:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:06:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:06:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:06:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:06:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:06:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:06:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:06:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:06:57.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.532780 29136 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:06:57.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.534768 29136 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:06:57.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.535317 29136 kserver.cc:163] Server-wide thread pool size limit: 3276
03:06:57.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.537915 29136 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:06:57.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.542181 29136 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:06:57.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.542382 29136 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:06:57.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.542578 29136 ts_tablet_manager.cc:616] Registered 0 tablets
03:06:57.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.542768 29136 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:06:57.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.681190 29136 rpc_server.cc:307] RPC server started. Bound to: 127.28.4.194:38547
03:06:57.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.681319 29272 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.4.194:38547 every 8 connection(s)
03:06:57.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.683791 29136 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-1/data/info.pb
03:06:57.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.689896 28691 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 29136
03:06:57.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.690503 28691 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-1/wal/instance
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.725415 28691 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-2/wal
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-2/data
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-2/logs
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-2/data/info.pb
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.4.195:0
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.4.195
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.4.195
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.4.254:40495,127.28.4.253:36601,127.28.4.252:43953
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.4.212:34923
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:06:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:06:57.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.729758 29273 heartbeater.cc:344] Connected to a master server at 127.28.4.252:43953
03:06:57.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.730430 29273 heartbeater.cc:461] Registering TS with master...
03:06:57.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.732498 29273 heartbeater.cc:507] Master 127.28.4.252:43953 requested a full tablet report, sending...
03:06:57.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.738870 28895 ts_manager.cc:194] Registered new tserver with Master: 4c52eee18c93483fa50cba6316720ce8 (127.28.4.194:38547)
03:06:57.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.740670 28895 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.4.194:53641
03:06:57.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.744606 29276 heartbeater.cc:344] Connected to a master server at 127.28.4.254:40495
03:06:57.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.744976 29276 heartbeater.cc:461] Registering TS with master...
03:06:57.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.745676 29276 heartbeater.cc:507] Master 127.28.4.254:40495 requested a full tablet report, sending...
03:06:57.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.748440 28738 ts_manager.cc:194] Registered new tserver with Master: 4c52eee18c93483fa50cba6316720ce8 (127.28.4.194:38547)
03:06:57.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.750877 29275 heartbeater.cc:344] Connected to a master server at 127.28.4.253:36601
03:06:57.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.751288 29275 heartbeater.cc:461] Registering TS with master...
03:06:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.752002 29275 heartbeater.cc:507] Master 127.28.4.253:36601 requested a full tablet report, sending...
03:06:57.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:57.754985 28814 ts_manager.cc:194] Registered new tserver with Master: 4c52eee18c93483fa50cba6316720ce8 (127.28.4.194:38547)
03:06:58.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:58.054420 29284 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:06:58.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:58.054996 29284 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:06:58.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:58.055467 29284 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:06:58.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:58.088646 29284 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:06:58.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:58.089447 29284 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.4.195
03:06:58.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:58.125413 29284 tablet_server_runner.cc:78] Tablet server non-default flags:
03:06:58.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:06:58.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.4.212:34923
03:06:58.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:06:58.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:06:58.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-2/data
03:06:58.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-2/wal
03:06:58.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.4.195:0
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-2/data/info.pb
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.4.195
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.4.254:40495,127.28.4.253:36601,127.28.4.252:43953
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.4.195
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-2/logs
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:58.126658 29284 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:06:58.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:58.128288 29284 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:06:58.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:58.140884 29290 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:06:58.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:58.744643 29273 heartbeater.cc:499] Master 127.28.4.252:43953 was elected leader, sending a full tablet report...
03:06:59.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:59.545081 29289 debug-util.cc:398] Leaking SignalData structure 0x7b080002eda0 after lost signal to thread 29284
03:06:59.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:59.704049 29289 kernel_stack_watchdog.cc:198] Thread 29284 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
03:06:59.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:06:59.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:06:59.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:06:59.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:06:59.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:58.141724 29291 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:06:59.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:59.705946 29284 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.565s user 0.000s sys 0.002s
03:06:59.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:59.706308 29284 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.566s user 0.000s sys 0.002s
03:06:59.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:59.706418 29292 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1564 milliseconds
03:06:59.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.708370 29284 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:06:59.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:06:59.708428 29293 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:06:59.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.711656 29284 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:06:59.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.713831 29284 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:06:59.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.715292 29284 hybrid_clock.cc:648] HybridClock initialized: now 1773976019715197 us; error 59 us; skew 500 ppm
03:06:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.716180 29284 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:06:59.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.722342 29284 webserver.cc:492] Webserver started at http://127.28.4.195:41605/ using document root <none> and password file <none>
03:06:59.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.723330 29284 fs_manager.cc:362] Metadata directory not provided
03:06:59.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.723575 29284 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:06:59.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.724045 29284 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:06:59.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.728423 29284 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-2/data/instance:
03:06:59.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "294ede7afe0946a9ba6d59712d70e0f1"
03:06:59.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:06:59 on dist-test-slave-sx75"
03:06:59.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.729557 29284 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-2/wal/instance:
03:06:59.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "294ede7afe0946a9ba6d59712d70e0f1"
03:06:59.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:06:59 on dist-test-slave-sx75"
03:06:59.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.736728 29284 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
03:06:59.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.742300 29301 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:06:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.743358 29284 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.743701 29284 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-2/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-2/wal
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "294ede7afe0946a9ba6d59712d70e0f1"
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:06:59 on dist-test-slave-sx75"
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.744032 29284 fs_report.cc:389] FS layout report
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-2/wal
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-2/wal
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-2/data/data
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:06:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:06:59.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.796473 29284 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:06:59.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.798151 29284 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:06:59.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.798696 29284 kserver.cc:163] Server-wide thread pool size limit: 3276
03:06:59.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.801375 29284 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:06:59.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.805706 29284 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:06:59.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.805938 29284 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:06:59.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.806246 29284 ts_tablet_manager.cc:616] Registered 0 tablets
03:06:59.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.806428 29284 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:06:59.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.949441 29284 rpc_server.cc:307] RPC server started. Bound to: 127.28.4.195:34977
03:06:59.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.949606 29415 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.4.195:34977 every 8 connection(s)
03:06:59.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.952183 29284 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-2/data/info.pb
03:06:59.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.961160 28691 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 29284
03:06:59.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:06:59.961736 28691 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster14533689530234602093/ts-2/wal/instance
03:07:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:00.001689 29416 heartbeater.cc:344] Connected to a master server at 127.28.4.252:43953
03:07:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:00.002197 29416 heartbeater.cc:461] Registering TS with master...
03:07:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:00.003222 29417 heartbeater.cc:344] Connected to a master server at 127.28.4.253:36601
03:07:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:00.003571 29417 heartbeater.cc:461] Registering TS with master...
03:07:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:00.003558 29416 heartbeater.cc:507] Master 127.28.4.252:43953 requested a full tablet report, sending...
03:07:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:00.003774 29418 heartbeater.cc:344] Connected to a master server at 127.28.4.254:40495
03:07:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:00.004137 29418 heartbeater.cc:461] Registering TS with master...
03:07:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:00.004606 29417 heartbeater.cc:507] Master 127.28.4.253:36601 requested a full tablet report, sending...
03:07:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:00.005089 29418 heartbeater.cc:507] Master 127.28.4.254:40495 requested a full tablet report, sending...
03:07:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:00.006497 28895 ts_manager.cc:194] Registered new tserver with Master: 294ede7afe0946a9ba6d59712d70e0f1 (127.28.4.195:34977)
03:07:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:00.007031 28814 ts_manager.cc:194] Registered new tserver with Master: 294ede7afe0946a9ba6d59712d70e0f1 (127.28.4.195:34977)
03:07:00.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:00.008384 28895 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.4.195:50335
03:07:00.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:00.010501 28738 ts_manager.cc:194] Registered new tserver with Master: 294ede7afe0946a9ba6d59712d70e0f1 (127.28.4.195:34977)
03:07:00.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:00.032562 28691 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
03:07:00.044 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:07:00.044 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
03:07:00.045 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.28.4.254"
port: 40495
}
bound_http_address {
host: "127.28.4.254"
port: 32833
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.28.4.253"
port: 36601
}
bound_http_address {
host: "127.28.4.253"
port: 34967
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.28.4.252"
port: 43953
}
bound_http_address {
host: "127.28.4.252"
port: 34417
}
}
}
03:07:00.045 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:07:00.046 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.28.4.193"
port: 40031
}
bound_http_address {
host: "127.28.4.193"
port: 43955
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.28.4.194"
port: 38547
}
bound_http_address {
host: "127.28.4.194"
port: 42499
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.28.4.195"
port: 34977
}
bound_http_address {
host: "127.28.4.195"
port: 41605
}
}
}
03:07:00.046 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:07:00.047 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster2771440754288159164/krb5kdc/krb5cc
03:07:00.048 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:07:00.048 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskoRAqye/test-tmp
03:07:00.049 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:07:00.049 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskoRAqye/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:07:00.051 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
03:07:00.394 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:00.395 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:07:00.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:00.399645 29427 mini_oidc.cc:150] Starting JWKS server
03:07:00.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:00.403661 29427 webserver.cc:492] Webserver started at http://127.0.0.1:41463/ using document root <none> and password file <none>
03:07:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:00.404225 29427 mini_oidc.cc:186] Starting OIDC Discovery server
03:07:00.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:00.406265 29427 webserver.cc:492] Webserver started at http://127.0.0.1:33167/ using document root <none> and password file <none>
03:07:00.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
03:07:00.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/principal' for realm 'KRBTEST.COM',
03:07:00.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
03:07:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:00 dist-test-slave-sx75 krb5kdc[29438](info): setting up network...
03:07:00.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
03:07:00.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:00 dist-test-slave-sx75 krb5kdc[29438](info): set up 2 sockets
03:07:00.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:00 dist-test-slave-sx75 krb5kdc[29438](info): commencing operation
03:07:00.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
03:07:01.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:01.012287 29416 heartbeater.cc:499] Master 127.28.4.252:43953 was elected leader, sending a full tablet report...
03:07:02.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:02.565184 29427 mini_kdc.cc:121] Time spent starting KDC: real 2.115s user 0.000s sys 0.020s
03:07:02.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
03:07:02.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:07:02.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
03:07:02.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
03:07:02.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:07:02.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
03:07:02.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
03:07:02.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:07:02.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
03:07:02.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:07:02.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/test-user.keytab.
03:07:02.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/test-user.keytab.
03:07:02.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:02 dist-test-slave-sx75 krb5kdc[29438](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976022, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:02.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:07:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-20T03:07:02Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
03:07:02.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-20T03:07:02Z Disabled control of system clock
03:07:02.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.188.254@KRBTEST.COM; defaulting to no policy
03:07:02.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:02.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.188.254@KRBTEST.COM" created.
03:07:02.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:02.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.188.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab.
03:07:02.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.188.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab.
03:07:02.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.188.254@KRBTEST.COM; defaulting to no policy
03:07:02.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:02.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.188.254@KRBTEST.COM" created.
03:07:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.188.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab.
03:07:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.188.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab.
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:02.811676 29427 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/logs
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/info.pb
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.254:39373
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.254
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.254
03:07:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:03.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:03.128211 29456 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:03.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:03.128759 29456 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:03.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:03.129384 29456 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:03.162119 29456 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:03.162365 29456 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:03.162629 29456 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:03.162909 29456 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:03.163113 29456 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:03.163317 29456 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:03.163466 29456 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41463/jwks.json
03:07:03.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:03.201383 29456 master_runner.cc:386] Master server non-default flags:
03:07:03.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:03.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:03.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:03.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:03.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data
03:07:03.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:03.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:03.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:03.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.254
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.254:39373
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/info.pb
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.254
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/logs
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:07:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:03.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:03.202775 29456 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:03.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:03.203811 29456 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:03.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:03.215647 29462 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:05.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:03.216374 29463 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:05.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.039016 29456 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.822s user 0.673s sys 1.097s
03:07:05.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:04.618839 29461 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 29456
03:07:05.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.040391 29456 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.824s user 0.673s sys 1.097s
03:07:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.041741 29465 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:05.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.044636 29464 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1823 milliseconds
03:07:05.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.044656 29456 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:05.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.045953 29456 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.048650 29456 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.050016 29456 hybrid_clock.cc:648] HybridClock initialized: now 1773976025049986 us; error 34 us; skew 500 ppm
03:07:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.050915 29456 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:05.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:05 dist-test-slave-sx75 krb5kdc[29438](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976025, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:05.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.056988 29456 init.cc:377] Logged in from keytab as kudu/127.28.188.254@KRBTEST.COM (short username kudu)
03:07:05.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.061875 29456 webserver.cc:492] Webserver started at http://127.28.188.254:35491/ using document root <none> and password file <none>
03:07:05.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.062888 29456 fs_manager.cc:362] Metadata directory not provided
03:07:05.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.063136 29456 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:05.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.063657 29456 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:07:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.068302 29456 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/instance:
03:07:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4854c286195847a89cac7db7426b1b6e"
03:07:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:05 on dist-test-slave-sx75"
03:07:05.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.069442 29456 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal/instance:
03:07:05.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4854c286195847a89cac7db7426b1b6e"
03:07:05.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:05 on dist-test-slave-sx75"
03:07:05.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.077479 29456 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
03:07:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.083187 29475 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:05.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.084436 29456 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
03:07:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.084826 29456 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4854c286195847a89cac7db7426b1b6e"
03:07:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:05 on dist-test-slave-sx75"
03:07:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.085144 29456 fs_report.cc:389] FS layout report
03:07:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/data
03:07:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:05.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:05.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:05.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:05.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:05.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:05.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.168541 29456 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.178294 29456 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:05.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.178763 29456 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:05.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.259943 29456 rpc_server.cc:307] RPC server started. Bound to: 127.28.188.254:39373
03:07:05.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.260030 29527 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.188.254:39373 every 8 connection(s)
03:07:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.263132 29456 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/info.pb
03:07:05.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.270079 29528 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:05.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.273066 29427 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 29456
03:07:05.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.273746 29427 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal/instance
03:07:05.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.284157 29528 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } has no permanent_uuid. Determining permanent_uuid...
03:07:05.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.188.253@KRBTEST.COM; defaulting to no policy
03:07:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.188.253@KRBTEST.COM" created.
03:07:05.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:05 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976025, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.254@KRBTEST.COM for kudu/127.28.188.254@KRBTEST.COM
03:07:05.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:05.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.188.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab.
03:07:05.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.188.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab.
03:07:05.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.326654 29528 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } has no permanent_uuid. Determining permanent_uuid...
03:07:05.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.329339 29477 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.334404 29528 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.253:44093: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:05.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.188.253@KRBTEST.COM; defaulting to no policy
03:07:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.188.253@KRBTEST.COM" created.
03:07:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.188.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab.
03:07:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.188.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab.
03:07:05.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.371487 29427 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:05.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:05.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:05.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/logs
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/info.pb
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.253:44093
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.253
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.253
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:05.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.384068 29528 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } attempt: 1
03:07:05.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.389467 29528 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.253:44093: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.457948 29528 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } attempt: 2
03:07:05.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.463320 29528 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.253:44093: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.554765 29528 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } attempt: 3
03:07:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.559252 29528 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.253:44093: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.702767 29528 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } attempt: 4
03:07:05.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.706894 29528 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.253:44093: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.717563 29537 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.718235 29537 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:05.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.719107 29537 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:05.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.753451 29537 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:05.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.753684 29537 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:05.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.753942 29537 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:05.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.754148 29537 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:05.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.754423 29537 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:05.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.754637 29537 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:05.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.754859 29537 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41463/jwks.json
03:07:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.796061 29537 master_runner.cc:386] Master server non-default flags:
03:07:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data
03:07:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.253
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.253:44093
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/info.pb
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.253
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/logs
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.797307 29537 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:05.798252 29537 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:05.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.809320 29544 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:06.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:06.006299 29528 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } attempt: 5
03:07:06.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:06.013217 29528 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.253:44093: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:06.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:06.560803 29528 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } attempt: 6
03:07:06.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:06.577209 29528 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.253:44093: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:07.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.810245 29545 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:07.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:05.814543 29547 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:07.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.156894 29546 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1342 milliseconds
03:07:07.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.157080 29537 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:07.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.158404 29537 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:07.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.161331 29537 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:07.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.162848 29537 hybrid_clock.cc:648] HybridClock initialized: now 1773976027162813 us; error 38 us; skew 500 ppm
03:07:07.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.163879 29537 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:07.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:07 dist-test-slave-sx75 krb5kdc[29438](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976027, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:07.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.170565 29537 init.cc:377] Logged in from keytab as kudu/127.28.188.253@KRBTEST.COM (short username kudu)
03:07:07.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.176483 29537 webserver.cc:492] Webserver started at http://127.28.188.253:36353/ using document root <none> and password file <none>
03:07:07.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.177554 29537 fs_manager.cc:362] Metadata directory not provided
03:07:07.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.177760 29537 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:07.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.178189 29537 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:07:07.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.183228 29537 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/instance:
03:07:07.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9362c8297aa946dfbcd513a95fa97f51"
03:07:07.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:07 on dist-test-slave-sx75"
03:07:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.184366 29537 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal/instance:
03:07:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9362c8297aa946dfbcd513a95fa97f51"
03:07:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:07 on dist-test-slave-sx75"
03:07:07.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.192631 29537 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.000s sys 0.008s
03:07:07.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.198124 29559 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:07.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.199292 29537 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.199673 29537 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9362c8297aa946dfbcd513a95fa97f51"
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:07 on dist-test-slave-sx75"
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.200022 29537 fs_report.cc:389] FS layout report
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/data
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:07.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.253748 29537 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:07.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.260083 29537 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:07.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.260608 29537 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:07.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.338644 29537 rpc_server.cc:307] RPC server started. Bound to: 127.28.188.253:44093
03:07:07.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.338796 29610 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.188.253:44093 every 8 connection(s)
03:07:07.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.341472 29537 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/info.pb
03:07:07.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.345389 29427 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 29537
03:07:07.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.345793 29427 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal/instance
03:07:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.348744 29611 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:07.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.188.252@KRBTEST.COM; defaulting to no policy
03:07:07.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.361248 29611 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } has no permanent_uuid. Determining permanent_uuid...
03:07:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.188.252@KRBTEST.COM" created.
03:07:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:07 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976027, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.253@KRBTEST.COM for kudu/127.28.188.254@KRBTEST.COM
03:07:07.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:07.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.188.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab.
03:07:07.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.188.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab.
03:07:07.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.404029 29611 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } has no permanent_uuid. Determining permanent_uuid...
03:07:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.188.252@KRBTEST.COM; defaulting to no policy
03:07:07.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:07.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.188.252@KRBTEST.COM" created.
03:07:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:07 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976027, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.253@KRBTEST.COM for kudu/127.28.188.253@KRBTEST.COM
03:07:07.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.431564 29611 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } has no permanent_uuid. Determining permanent_uuid...
03:07:07.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.433795 29561 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:07.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.437822 29611 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.252:43955: Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.188.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab.
03:07:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.188.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab.
03:07:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.443535 29427 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data
03:07:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/logs
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/info.pb
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.252:43955
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.252
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.252
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:07.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.487341 29611 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } attempt: 1
03:07:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.492152 29611 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.252:43955: Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:07.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.560577 29611 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } attempt: 2
03:07:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.565019 29611 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.252:43955: Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.637845 29528 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } attempt: 7
03:07:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:07 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976025, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.254@KRBTEST.COM for kudu/127.28.188.253@KRBTEST.COM
03:07:07.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.656409 29611 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } attempt: 3
03:07:07.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.661402 29611 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.252:43955: Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:07.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.661449 29528 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } has no permanent_uuid. Determining permanent_uuid...
03:07:07.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.666908 29528 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.252:43955: Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:07.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.725404 29528 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } attempt: 1
03:07:07.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.729781 29528 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.252:43955: Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:07.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.768662 29621 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:07.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.769248 29621 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:07.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.769876 29621 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:07.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.804857 29611 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } attempt: 4
03:07:07.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.805481 29621 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:07.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.805768 29621 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:07.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.806172 29621 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:07.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.806551 29621 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:07.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.806917 29621 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:07.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.807268 29621 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:07.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.807498 29621 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41463/jwks.json
03:07:07.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.809610 29611 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.252:43955: Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.811314 29528 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } attempt: 2
03:07:07.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.815369 29528 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.252:43955: Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:07.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.848378 29621 master_runner.cc:386] Master server non-default flags:
03:07:07.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:07.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:07.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:07.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:07.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.252
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.252:43955
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/info.pb
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.252
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/logs
03:07:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:07:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:07:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:07:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.850049 29621 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:07.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.851364 29621 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:07.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.861960 29629 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:07.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:07.900904 29528 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } attempt: 3
03:07:07.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.905797 29528 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.252:43955: Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:08.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:08.046213 29528 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } attempt: 4
03:07:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:08.052978 29528 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.252:43955: Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:08.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:08.109047 29611 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } attempt: 5
03:07:08.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:08.115613 29611 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.252:43955: Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:08.336628 29528 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } attempt: 5
03:07:08.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:08.351512 29528 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.252:43955: Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:08.663632 29611 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } attempt: 6
03:07:08.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:08.674921 29611 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.252:43955: Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:08.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:08.904142 29528 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } attempt: 6
03:07:08.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:08.915966 29528 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.188.252:43955: Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:09.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:07.863751 29630 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:09.193715 29632 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:09.196884 29631 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1328 milliseconds
03:07:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.197038 29621 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.198271 29621 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:09.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.201440 29621 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:09.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.202858 29621 hybrid_clock.cc:648] HybridClock initialized: now 1773976029202797 us; error 74 us; skew 500 ppm
03:07:09.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.203694 29621 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:09.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:09 dist-test-slave-sx75 krb5kdc[29438](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976029, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:09.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.209767 29621 init.cc:377] Logged in from keytab as kudu/127.28.188.252@KRBTEST.COM (short username kudu)
03:07:09.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.215312 29621 webserver.cc:492] Webserver started at http://127.28.188.252:37157/ using document root <none> and password file <none>
03:07:09.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.216250 29621 fs_manager.cc:362] Metadata directory not provided
03:07:09.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.216512 29621 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:09.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.217074 29621 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:07:09.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.221545 29621 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/instance:
03:07:09.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58b878598d2e431ba4a31b17f243118d"
03:07:09.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:09 on dist-test-slave-sx75"
03:07:09.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.222667 29621 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal/instance:
03:07:09.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58b878598d2e431ba4a31b17f243118d"
03:07:09.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:09 on dist-test-slave-sx75"
03:07:09.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.230947 29621 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.003s sys 0.004s
03:07:09.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.239091 29647 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:09.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.240466 29621 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
03:07:09.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.240869 29621 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:09.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58b878598d2e431ba4a31b17f243118d"
03:07:09.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:09 on dist-test-slave-sx75"
03:07:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.241290 29621 fs_report.cc:389] FS layout report
03:07:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/data
03:07:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.300520 29621 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:09.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.306329 29621 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:09.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.306792 29621 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.397074 29621 rpc_server.cc:307] RPC server started. Bound to: 127.28.188.252:43955
03:07:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.397210 29699 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.188.252:43955 every 8 connection(s)
03:07:09.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.400219 29621 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/info.pb
03:07:09.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.401059 29427 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 29621
03:07:09.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.401535 29427 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal/instance
03:07:09.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.409345 29700 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:09.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.427031 29700 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } has no permanent_uuid. Determining permanent_uuid...
03:07:09.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:09 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976029, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.252@KRBTEST.COM for kudu/127.28.188.254@KRBTEST.COM
03:07:09.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:09 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976022, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.188.252@KRBTEST.COM
03:07:09.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.463203 29700 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } has no permanent_uuid. Determining permanent_uuid...
03:07:09.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:09 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976029, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.252@KRBTEST.COM for kudu/127.28.188.253@KRBTEST.COM
03:07:09.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:09 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976022, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.188.253@KRBTEST.COM
03:07:09.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.489604 29700 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } has no permanent_uuid. Determining permanent_uuid...
03:07:09.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:09 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976029, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.252@KRBTEST.COM for kudu/127.28.188.252@KRBTEST.COM
03:07:09.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:09 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976022, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.188.254@KRBTEST.COM
03:07:09.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.528455 29700 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Bootstrap starting.
03:07:09.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.532598 29427 external_mini_cluster.cc:949] 0 TS(s) registered with all masters
03:07:09.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.536196 29700 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Neither blocks nor log segments found. Creating new log.
03:07:09.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.538548 29700 log.cc:826] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Log is configured to *not* fsync() on all Append() calls
03:07:09.544 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:09.544 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
03:07:09.545 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
03:07:09.545 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
03:07:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.546000 29700 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: No bootstrap required, opened a new log
03:07:09.546 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.28.188.254"
port: 39373
}
bound_http_address {
host: "127.28.188.254"
port: 35491
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.28.188.253"
port: 44093
}
bound_http_address {
host: "127.28.188.253"
port: 36353
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.28.188.252"
port: 43955
}
bound_http_address {
host: "127.28.188.252"
port: 37157
}
}
}
03:07:09.546 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:07:09.547 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:09.547 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:07:09.547 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:07:09.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.571573 29700 raft_consensus.cc:359] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:09.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.572569 29700 raft_consensus.cc:385] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:09.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.572916 29700 raft_consensus.cc:740] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58b878598d2e431ba4a31b17f243118d, State: Initialized, Role: FOLLOWER
03:07:09.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.573915 29700 consensus_queue.cc:260] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:09.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:09 dist-test-slave-sx75 krb5kdc[29438](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976029, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:09.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.577457 29711 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:09.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:07:09.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.578331 29711 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: This master's current role is: FOLLOWER
03:07:09.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.579345 29700 sys_catalog.cc:565] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: configured and running, proceeding with master startup.
03:07:09.583 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:09.585 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc
03:07:09.586 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:07:09.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:09.606437 29722 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:07:09.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:09.606866 29722 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:07:09.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.602983 29621 master_runner.cc:186] Error getting master registration for 127.28.188.254:39373: OK, instance_id { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" instance_seqno: 1773976025238053 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:07:09.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.625780 29621 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:09.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.735518 29611 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } attempt: 7
03:07:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:09 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976027, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.253@KRBTEST.COM for kudu/127.28.188.252@KRBTEST.COM
03:07:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.764142 29611 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Bootstrap starting.
03:07:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.769765 29611 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Neither blocks nor log segments found. Creating new log.
03:07:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.771729 29611 log.cc:826] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Log is configured to *not* fsync() on all Append() calls
03:07:09.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.776639 29611 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: No bootstrap required, opened a new log
03:07:09.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.795929 29611 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:09.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.796562 29611 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:09.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.796762 29611 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9362c8297aa946dfbcd513a95fa97f51, State: Initialized, Role: FOLLOWER
03:07:09.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.797494 29611 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.800617 29726 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.801460 29726 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: This master's current role is: FOLLOWER
03:07:09.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.802474 29611 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: configured and running, proceeding with master startup.
03:07:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:09.813220 29653 tablet.cc:2379] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:07:09.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:09.834640 29738 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:07:09.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:09.835171 29738 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:07:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.831809 29537 master_runner.cc:186] Error getting master registration for 127.28.188.254:39373: OK, instance_id { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" instance_seqno: 1773976025238053 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:07:09.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.852973 29537 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:09.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.916395 29726 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:07:09.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.917017 29726 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:09.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.921283 29726 leader_election.cc:290] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4854c286195847a89cac7db7426b1b6e (127.28.188.254:39373), 58b878598d2e431ba4a31b17f243118d (127.28.188.252:43955)
03:07:09.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.921366 29674 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9362c8297aa946dfbcd513a95fa97f51" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "58b878598d2e431ba4a31b17f243118d" is_pre_election: true
03:07:09.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.922333 29674 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9362c8297aa946dfbcd513a95fa97f51 in term 0.
03:07:09.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.923650 29561 leader_election.cc:304] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 58b878598d2e431ba4a31b17f243118d, 9362c8297aa946dfbcd513a95fa97f51; no voters:
03:07:09.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.924540 29726 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:07:09.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.924897 29726 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:07:09.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.925233 29726 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 0 FOLLOWER]: Advancing to term 1
03:07:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.933890 29726 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:09.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.936590 29726 leader_election.cc:290] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [CANDIDATE]: Term 1 election: Requested vote from peers 4854c286195847a89cac7db7426b1b6e (127.28.188.254:39373), 58b878598d2e431ba4a31b17f243118d (127.28.188.252:43955)
03:07:09.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.937808 29674 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9362c8297aa946dfbcd513a95fa97f51" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "58b878598d2e431ba4a31b17f243118d"
03:07:09.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.938387 29674 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 0 FOLLOWER]: Advancing to term 1
03:07:09.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.945664 29674 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9362c8297aa946dfbcd513a95fa97f51 in term 1.
03:07:09.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.946938 29561 leader_election.cc:304] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 58b878598d2e431ba4a31b17f243118d, 9362c8297aa946dfbcd513a95fa97f51; no voters:
03:07:09.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.948068 29726 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 1 FOLLOWER]: Leader election won for term 1
03:07:09.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.954787 29528 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } attempt: 7
03:07:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.959398 29726 raft_consensus.cc:697] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 1 LEADER]: Becoming Leader. State: Replica: 9362c8297aa946dfbcd513a95fa97f51, State: Running, Role: LEADER
03:07:09.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.960402 29726 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:09 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976025, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.254@KRBTEST.COM for kudu/127.28.188.252@KRBTEST.COM
03:07:09.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.987874 29740 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9362c8297aa946dfbcd513a95fa97f51. Latest consensus state: current_term: 1 leader_uuid: "9362c8297aa946dfbcd513a95fa97f51" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.988716 29740 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: This master's current role is: LEADER
03:07:09.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:09.994298 29743 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:07:10.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.012414 29743 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:07:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.020126 29528 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Bootstrap starting.
03:07:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:10.020624 29565 tablet.cc:2379] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:07:10.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.028491 29528 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Neither blocks nor log segments found. Creating new log.
03:07:10.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.031235 29528 log.cc:826] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Log is configured to *not* fsync() on all Append() calls
03:07:10.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.048000 29528 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: No bootstrap required, opened a new log
03:07:10.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.052858 29674 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 1 FOLLOWER]: Refusing update from remote peer 9362c8297aa946dfbcd513a95fa97f51: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:07:10.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.055038 29726 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [LEADER]: Connected to new peer: Peer: permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:07:10.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.107257 29711 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9362c8297aa946dfbcd513a95fa97f51. Latest consensus state: current_term: 1 leader_uuid: "9362c8297aa946dfbcd513a95fa97f51" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:10.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.108603 29711 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: This master's current role is: FOLLOWER
03:07:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.119266 29726 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9362c8297aa946dfbcd513a95fa97f51" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.120044 29726 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: This master's current role is: LEADER
03:07:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.123996 29743 catalog_manager.cc:1357] Generated new cluster ID: 7b1010dc34f4435aa7dbf3461c94b984
03:07:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.124297 29743 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:07:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.125128 29746 mvcc.cc:204] Tried to move back new op lower bound from 7266205819070820352 to 7266205818732343296. Current Snapshot: MvccSnapshot[applied={T|T < 7266205819070820352}]
03:07:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.134421 29711 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9362c8297aa946dfbcd513a95fa97f51" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:10.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.136812 29711 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: This master's current role is: FOLLOWER
03:07:10.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.147651 29528 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:10.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.148823 29528 raft_consensus.cc:385] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:10.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.149271 29528 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4854c286195847a89cac7db7426b1b6e, State: Initialized, Role: FOLLOWER
03:07:10.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.150419 29528 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.160485 29750 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:10.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.165802 29750 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: This master's current role is: FOLLOWER
03:07:10.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.165992 29500 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 0 FOLLOWER]: Advancing to term 1
03:07:10.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.165350 29528 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: configured and running, proceeding with master startup.
03:07:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.170225 29501 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9362c8297aa946dfbcd513a95fa97f51" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4854c286195847a89cac7db7426b1b6e"
03:07:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.171975 29502 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9362c8297aa946dfbcd513a95fa97f51" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4854c286195847a89cac7db7426b1b6e" is_pre_election: true
03:07:10.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.179167 29500 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 1 FOLLOWER]: Refusing update from remote peer 9362c8297aa946dfbcd513a95fa97f51: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:07:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.181850 29740 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:07:10.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:10.190016 29481 tablet.cc:2379] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:07:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.210233 29743 catalog_manager.cc:1380] Generated new certificate authority record
03:07:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.212793 29743 catalog_manager.cc:1514] Loading token signing keys...
03:07:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.260201 29743 catalog_manager.cc:6044] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Generated new TSK 0
03:07:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.261209 29743 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:07:10.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.269225 29740 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9362c8297aa946dfbcd513a95fa97f51" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:10.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.269925 29740 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: This master's current role is: LEADER
03:07:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.272606 29750 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9362c8297aa946dfbcd513a95fa97f51. Latest consensus state: current_term: 1 leader_uuid: "9362c8297aa946dfbcd513a95fa97f51" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.273311 29750 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: This master's current role is: FOLLOWER
03:07:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.281669 29763 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9362c8297aa946dfbcd513a95fa97f51" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.282482 29763 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: This master's current role is: FOLLOWER
03:07:10.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.320614 29769 catalog_manager.cc:1269] Loaded cluster ID: 7b1010dc34f4435aa7dbf3461c94b984
03:07:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.321108 29769 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: loading cluster ID for follower catalog manager: success
03:07:10.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.328676 29769 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: acquiring CA information for follower catalog manager: success
03:07:10.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.334995 29769 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:10.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:10.456408 29695 messenger.cc:194] Invalid argument: socket address is ambiguous: no data on pending connections for acceptor pool at 127.28.188.252:43955
03:07:10.587 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.188.254:39373
03:07:10.588 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:07:10.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.588742 29427 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 29456
03:07:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.612738 29722 catalog_manager.cc:1269] Loaded cluster ID: 7b1010dc34f4435aa7dbf3461c94b984
03:07:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.613274 29722 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: loading cluster ID for follower catalog manager: success
03:07:10.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.624114 29722 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: acquiring CA information for follower catalog manager: success
03:07:10.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.629743 29722 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:10.631 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:10.631 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.188.252:43955
03:07:10.632 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:07:10.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.634122 29427 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 29621
03:07:10.664 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:10.665 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.188.253:44093
03:07:10.665 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:07:10.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.665591 29427 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 29537
03:07:10.696 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:10.696 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.188.254:39373
03:07:10.696 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:10.697918 29427 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/logs
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/info.pb
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.254:39373
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.254
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35491
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.254
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:11.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:11.033825 29772 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:11.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:11.034373 29772 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:11.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:11.035109 29772 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:11.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:11.068553 29772 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:11.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:11.068797 29772 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:11.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:11.069062 29772 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:11.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:11.069271 29772 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:11.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:11.069573 29772 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:11.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:11.069783 29772 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:11.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:11.069936 29772 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41463/jwks.json
03:07:11.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:11.111344 29772 master_runner.cc:386] Master server non-default flags:
03:07:11.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:11.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:11.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:11.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:11.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data
03:07:11.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:11.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:11.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:11.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:11.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:11.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.254
03:07:11.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.254:39373
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/info.pb
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.254
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35491
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/logs
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:07:11.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:07:11.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:11.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:11.112583 29772 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:11.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:11.113518 29772 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:11.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:11.124836 29778 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:12.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:11.130359 29781 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:12.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:11.126150 29779 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:12.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.499786 29772 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:12.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:12.500368 29780 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1372 milliseconds
03:07:12.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.508548 29772 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:12.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.512318 29772 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:12.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.513940 29772 hybrid_clock.cc:648] HybridClock initialized: now 1773976032513856 us; error 79 us; skew 500 ppm
03:07:12.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.515213 29772 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:12.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:12 dist-test-slave-sx75 krb5kdc[29438](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976032, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:12.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.523211 29772 init.cc:377] Logged in from keytab as kudu/127.28.188.254@KRBTEST.COM (short username kudu)
03:07:12.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.531231 29772 webserver.cc:492] Webserver started at http://127.28.188.254:35491/ using document root <none> and password file <none>
03:07:12.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.532449 29772 fs_manager.cc:362] Metadata directory not provided
03:07:12.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.532749 29772 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:12.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.542174 29772 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.005s sys 0.001s
03:07:12.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.546952 29790 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:12.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.548182 29772 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
03:07:12.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.548591 29772 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:12.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4854c286195847a89cac7db7426b1b6e"
03:07:12.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:05 on dist-test-slave-sx75"
03:07:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.550547 29772 fs_report.cc:389] FS layout report
03:07:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/data
03:07:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:12.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.608600 29772 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:12.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.619320 29772 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:12.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.619861 29772 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:12.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.696928 29772 rpc_server.cc:307] RPC server started. Bound to: 127.28.188.254:39373
03:07:12.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.696965 29841 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.188.254:39373 every 8 connection(s)
03:07:12.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.700618 29772 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/info.pb
03:07:12.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.701337 29427 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 29772
03:07:12.701 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:12.702 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.188.252:43955
03:07:12.702 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:07:12.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.703413 29427 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:12.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:12.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:12.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/logs
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/info.pb
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.252:43955
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.252
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37157
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.252
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:12.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.714984 29842 sys_catalog.cc:263] Verifying existing consensus state
03:07:12.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.724673 29842 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Bootstrap starting.
03:07:12.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.774019 29842 log.cc:826] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Log is configured to *not* fsync() on all Append() calls
03:07:12.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.793080 29842 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:07:12.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.794080 29842 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Bootstrap complete.
03:07:12.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.820704 29842 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:12.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.823413 29842 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4854c286195847a89cac7db7426b1b6e, State: Initialized, Role: FOLLOWER
03:07:12.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.824401 29842 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:12.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.827682 29846 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:12.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.828569 29846 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: This master's current role is: FOLLOWER
03:07:12.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.829608 29842 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: configured and running, proceeding with master startup.
03:07:12.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.867357 29857 catalog_manager.cc:1269] Loaded cluster ID: 7b1010dc34f4435aa7dbf3461c94b984
03:07:12.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.867676 29857 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: loading cluster ID for follower catalog manager: success
03:07:12.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.877882 29857 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: acquiring CA information for follower catalog manager: success
03:07:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:12 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976032, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.254@KRBTEST.COM for kudu/127.28.188.254@KRBTEST.COM
03:07:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.883544 29857 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.900604 29772 master_runner.cc:186] Error getting master registration for 127.28.188.253:44093: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111),
03:07:12.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:12.923240 29772 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:13.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:13.169940 29844 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:13.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:13.170696 29844 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:13.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:13.171725 29844 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:13.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:13.209939 29844 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:13.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:13.210227 29844 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:13.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:13.210767 29844 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:13.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:13.211162 29844 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:13.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:13.211491 29844 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:13.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:13.212057 29844 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:13.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:13.212347 29844 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41463/jwks.json
03:07:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:13.258690 29844 master_runner.cc:386] Master server non-default flags:
03:07:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data
03:07:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.252
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.252:43955
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/info.pb
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.252
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37157
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/logs
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:13.260442 29844 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:13.261674 29844 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:13.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:13.273331 29866 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:14.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.375486 29874 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:07:14.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.376536 29874 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:14.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.380061 29874 leader_election.cc:290] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093), 58b878598d2e431ba4a31b17f243118d (127.28.188.252:43955)
03:07:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:14.398093 29792 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:14.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:14.427567 29792 leader_election.cc:336] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093): Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:14.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:14.428341 29792 leader_election.cc:336] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 58b878598d2e431ba4a31b17f243118d (127.28.188.252:43955): Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:14.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.436504 29792 leader_election.cc:304] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4854c286195847a89cac7db7426b1b6e; no voters: 58b878598d2e431ba4a31b17f243118d, 9362c8297aa946dfbcd513a95fa97f51
03:07:14.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.437417 29874 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
03:07:14.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:13.279095 29869 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:14.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:13.280786 29844 server_base.cc:1061] running on GCE node
03:07:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:13.278409 29867 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:14.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.654392 29844 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.657716 29844 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:14.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.659147 29844 hybrid_clock.cc:648] HybridClock initialized: now 1773976034659111 us; error 53 us; skew 500 ppm
03:07:14.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.660107 29844 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:14.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:14 dist-test-slave-sx75 krb5kdc[29438](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976034, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:14.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.666599 29844 init.cc:377] Logged in from keytab as kudu/127.28.188.252@KRBTEST.COM (short username kudu)
03:07:14.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.672803 29844 webserver.cc:492] Webserver started at http://127.28.188.252:37157/ using document root <none> and password file <none>
03:07:14.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.673774 29844 fs_manager.cc:362] Metadata directory not provided
03:07:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.674043 29844 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:14.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.683674 29844 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
03:07:14.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.689349 29884 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:14.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.690629 29844 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
03:07:14.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.691136 29844 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:14.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58b878598d2e431ba4a31b17f243118d"
03:07:14.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:09 on dist-test-slave-sx75"
03:07:14.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.693989 29844 fs_report.cc:389] FS layout report
03:07:14.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:14.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:14.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:14.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/data
03:07:14.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:14.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:14.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:14.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:14.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:14.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:14.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:14.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:14.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:14.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:14.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:14.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.751536 29844 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:14.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.757437 29844 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:14.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.757982 29844 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:14.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.849989 29844 rpc_server.cc:307] RPC server started. Bound to: 127.28.188.252:43955
03:07:14.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.850028 29936 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.188.252:43955 every 8 connection(s)
03:07:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.853925 29844 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/info.pb
03:07:14.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.854651 29427 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 29844
03:07:14.855 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:14.855 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.188.253:44093
03:07:14.855 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:07:14.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.857076 29427 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:14.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:14.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:14.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data
03:07:14.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:14.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:14.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:14.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:14.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:14.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:14.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/logs
03:07:14.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/info.pb
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.253:44093
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.253
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36353
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.253
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:14.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.871614 29937 sys_catalog.cc:263] Verifying existing consensus state
03:07:14.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.877803 29937 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Bootstrap starting.
03:07:14.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.922336 29937 log.cc:826] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Log is configured to *not* fsync() on all Append() calls
03:07:14.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.940855 29937 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:07:14.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.941804 29937 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Bootstrap complete.
03:07:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.973106 29937 raft_consensus.cc:359] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.975605 29937 raft_consensus.cc:740] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58b878598d2e431ba4a31b17f243118d, State: Initialized, Role: FOLLOWER
03:07:14.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.976827 29937 consensus_queue.cc:260] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.980130 29941 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:14.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.981083 29941 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: This master's current role is: FOLLOWER
03:07:14.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:14.982237 29937 sys_catalog.cc:565] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: configured and running, proceeding with master startup.
03:07:15.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:15.024041 29952 catalog_manager.cc:1269] Loaded cluster ID: 7b1010dc34f4435aa7dbf3461c94b984
03:07:15.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:15.024372 29952 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: loading cluster ID for follower catalog manager: success
03:07:15.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:15.032433 29952 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: acquiring CA information for follower catalog manager: success
03:07:15.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:15.038210 29952 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:15.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:15 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976034, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.252@KRBTEST.COM for kudu/127.28.188.254@KRBTEST.COM
03:07:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:15.079808 29844 master_runner.cc:186] Error getting master registration for 127.28.188.253:44093: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111),
03:07:15.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:15.106012 29844 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:15.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:15.306373 29939 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:15.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:15.307116 29939 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:15.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:15.307914 29939 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:15.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:15.348560 29939 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:15.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:15.348939 29939 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:15.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:15.349512 29939 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:15.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:15.349926 29939 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:15.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:15.350306 29939 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:15.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:15.350693 29939 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:15.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:15.351033 29939 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41463/jwks.json
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:15.418880 29939 master_runner.cc:386] Master server non-default flags:
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.253
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.253:44093
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/info.pb
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.253
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36353
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/logs
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:07:15.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:07:15.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:15.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:15.420703 29939 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:15.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:15.422081 29939 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:15.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:15.436340 29963 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:15.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:15.879496 29933 messenger.cc:194] Invalid argument: socket address is ambiguous: no data on pending connections for acceptor pool at 127.28.188.252:43955
03:07:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.136634 29970 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:07:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.137130 29970 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:16.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.159547 29970 leader_election.cc:290] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093), 58b878598d2e431ba4a31b17f243118d (127.28.188.252:43955)
03:07:16.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:16.169730 29792 leader_election.cc:336] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093): Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:16.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.207727 29911 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4854c286195847a89cac7db7426b1b6e" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "58b878598d2e431ba4a31b17f243118d" is_pre_election: true
03:07:16.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.208729 29911 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4854c286195847a89cac7db7426b1b6e in term 1.
03:07:16.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.210414 29792 leader_election.cc:304] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4854c286195847a89cac7db7426b1b6e, 58b878598d2e431ba4a31b17f243118d; no voters: 9362c8297aa946dfbcd513a95fa97f51
03:07:16.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.211580 29970 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 1 FOLLOWER]: Leader pre-election won for term 2
03:07:16.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.211921 29970 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:07:16.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.212260 29970 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 1 FOLLOWER]: Advancing to term 2
03:07:16.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.225441 29970 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:16.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.231429 29970 leader_election.cc:290] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 2 election: Requested vote from peers 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093), 58b878598d2e431ba4a31b17f243118d (127.28.188.252:43955)
03:07:16.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:16.232713 29792 leader_election.cc:336] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093): Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:16.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.234470 29911 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4854c286195847a89cac7db7426b1b6e" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "58b878598d2e431ba4a31b17f243118d"
03:07:16.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.235037 29911 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 1 FOLLOWER]: Advancing to term 2
03:07:16.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.248163 29911 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4854c286195847a89cac7db7426b1b6e in term 2.
03:07:16.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.249396 29792 leader_election.cc:304] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4854c286195847a89cac7db7426b1b6e, 58b878598d2e431ba4a31b17f243118d; no voters: 9362c8297aa946dfbcd513a95fa97f51
03:07:16.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.250427 29970 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 2 FOLLOWER]: Leader election won for term 2
03:07:16.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.253306 29970 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 2 LEADER]: Becoming Leader. State: Replica: 4854c286195847a89cac7db7426b1b6e, State: Running, Role: LEADER
03:07:16.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.254277 29970 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:16.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.282555 29976 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4854c286195847a89cac7db7426b1b6e. Latest consensus state: current_term: 2 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:16.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.283584 29976 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: This master's current role is: LEADER
03:07:16.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.298571 29978 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:07:16.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.316027 29978 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:07:16.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.321524 29978 catalog_manager.cc:1269] Loaded cluster ID: 7b1010dc34f4435aa7dbf3461c94b984
03:07:16.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.321885 29978 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:07:16.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.343822 29978 catalog_manager.cc:1514] Loading token signing keys...
03:07:16.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.356824 29978 catalog_manager.cc:6055] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Loaded TSK: 0
03:07:16.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.358538 29978 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:07:16.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.743628 29911 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 2 FOLLOWER]: Refusing update from remote peer 4854c286195847a89cac7db7426b1b6e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
03:07:16.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.745952 29976 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [LEADER]: Connected to new peer: Peer: permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
03:07:16.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.826308 29984 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:16.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.827518 29984 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: This master's current role is: LEADER
03:07:16.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.809888 29979 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4854c286195847a89cac7db7426b1b6e. Latest consensus state: current_term: 2 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:16.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.832969 29979 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: This master's current role is: FOLLOWER
03:07:16.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.857069 29979 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:16.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:16.858315 29979 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: This master's current role is: FOLLOWER
03:07:16.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:16.862857 29792 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e -> Peer 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093): Couldn't send request to peer 9362c8297aa946dfbcd513a95fa97f51. Status: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:07:17.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:15.438930 29964 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:17.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:16.838690 29962 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 29939
03:07:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:17.331036 29962 kernel_stack_watchdog.cc:198] Thread 29939 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
03:07:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:07:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:07:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:07:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:07:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:17.331496 29939 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.895s user 0.532s sys 1.062s
03:07:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:17.331976 29939 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.896s user 0.532s sys 1.063s
03:07:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:17.332461 29966 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:17.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:17.335281 29965 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1890 milliseconds
03:07:17.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.335815 29939 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.337067 29939 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:17.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.339761 29939 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:17.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.341130 29939 hybrid_clock.cc:648] HybridClock initialized: now 1773976037341080 us; error 77 us; skew 500 ppm
03:07:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.342084 29939 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:17.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:17 dist-test-slave-sx75 krb5kdc[29438](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976037, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:17.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.348706 29939 init.cc:377] Logged in from keytab as kudu/127.28.188.253@KRBTEST.COM (short username kudu)
03:07:17.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.356285 29939 webserver.cc:492] Webserver started at http://127.28.188.253:36353/ using document root <none> and password file <none>
03:07:17.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.357592 29939 fs_manager.cc:362] Metadata directory not provided
03:07:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.357908 29939 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:17.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.367309 29939 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.002s
03:07:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.372238 29994 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:17.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.373574 29939 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.004s
03:07:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.373956 29939 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9362c8297aa946dfbcd513a95fa97f51"
03:07:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:07 on dist-test-slave-sx75"
03:07:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.376106 29939 fs_report.cc:389] FS layout report
03:07:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/data
03:07:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:17.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.464447 29939 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:17.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.470738 29939 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:17.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.471310 29939 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.599565 30051 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.188.253:44093 every 8 connection(s)
03:07:17.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.599519 29939 rpc_server.cc:307] RPC server started. Bound to: 127.28.188.253:44093
03:07:17.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.603669 29939 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/info.pb
03:07:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.605731 29427 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 29939
03:07:17.606 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:17.609 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.188.254:39373(127.28.188.254:39373)] connecting to peer
03:07:17.610 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.188.252:43955(127.28.188.252:43955)] connecting to peer
03:07:17.611 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.28.188.254:39373(127.28.188.254:39373)] Successfully connected to peer
03:07:17.613 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.188.253:44093(127.28.188.253:44093)] connecting to peer
03:07:17.614 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
03:07:17.615 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc
03:07:17.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:17 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773976029, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:17.621 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:07:17.621 [INFO - kudu-nio-0] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
03:07:17.623 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.28.188.254
03:07:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.620218 30052 sys_catalog.cc:263] Verifying existing consensus state
03:07:17.627 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.28.188.253:44093(127.28.188.253:44093)] Successfully connected to peer
03:07:17.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.636353 30052 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Bootstrap starting.
03:07:17.640 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.28.188.252:43955(127.28.188.252:43955)] Successfully connected to peer
03:07:17.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:17 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773976029, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.188.254@KRBTEST.COM
03:07:17.646 [DEBUG - kudu-nio-1] (SecurityContext.java:222) Refreshing Kerberos credentials...
03:07:17.647 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc
03:07:17.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:17 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773976029, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:17.657 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:07:17.657 [INFO - kudu-nio-1] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
03:07:17.658 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.28.188.252
03:07:17.660 [DEBUG - kudu-nio-2] (SecurityContext.java:222) Refreshing Kerberos credentials...
03:07:17.660 [DEBUG - kudu-nio-2] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc
03:07:17.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:17 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773976029, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:17.665 [DEBUG - kudu-nio-2] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:07:17.665 [INFO - kudu-nio-2] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
03:07:17.666 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.28.188.253
03:07:17.673 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x71ed9b3f, L:/127.0.0.1:56968 - R:/127.28.188.254:39373] using SASL/GSSAPI
03:07:17.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:17 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773976029, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.188.253@KRBTEST.COM
03:07:17.714 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:07:17.714 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=108), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.188.254:39373, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.188.252:43955, [5ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.188.253:44093, [105ms] Sub RPC ConnectToMaster: received response from server master-127.28.188.254:39373: OK, deferred=null) after lookup
03:07:17.742 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x5740dfb8, L:/127.0.0.1:56212 - R:/127.28.188.253:44093] using SASL/GSSAPI
03:07:17.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:17 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773976029, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.188.252@KRBTEST.COM
03:07:17.757 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.28.188.253:44093: Catalog manager is not initialized. State: Starting
03:07:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.759106 30052 log.cc:826] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Log is configured to *not* fsync() on all Append() calls
03:07:17.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.786741 30052 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:07:17.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.788233 30052 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Bootstrap complete.
03:07:17.794 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x744aef38, L:/127.0.0.1:59920 - R:/127.28.188.252:43955] using SASL/GSSAPI
03:07:17.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.819821 30052 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:17.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.823172 30052 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9362c8297aa946dfbcd513a95fa97f51, State: Initialized, Role: FOLLOWER
03:07:17.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.824235 30052 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.839375 30063 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.840324 30063 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: This master's current role is: FOLLOWER
03:07:17.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.854435 30052 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: configured and running, proceeding with master startup.
03:07:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.935927 30076 catalog_manager.cc:1269] Loaded cluster ID: 7b1010dc34f4435aa7dbf3461c94b984
03:07:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.936231 30076 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: loading cluster ID for follower catalog manager: success
03:07:17.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:17 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976032, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.254@KRBTEST.COM for kudu/127.28.188.253@KRBTEST.COM
03:07:17.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.943472 30076 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: acquiring CA information for follower catalog manager: success
03:07:17.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.949018 30076 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:17.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:17 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976037, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.253@KRBTEST.COM for kudu/127.28.188.254@KRBTEST.COM
03:07:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.981762 29772 master_runner.cc:232] Existing masters have differing terms: 1 vs 2
03:07:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.981657 30026 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 1 FOLLOWER]: Advancing to term 2
03:07:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:17.982224 29772 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:18.001677 30081 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:18.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:18.002668 30081 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: This master's current role is: FOLLOWER
03:07:18.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:18.008754 30063 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4854c286195847a89cac7db7426b1b6e. Latest consensus state: current_term: 2 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:18.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:18.009420 30063 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: This master's current role is: FOLLOWER
03:07:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:18.417668 29997 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:18.418452 29997 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: This master's current role is: LEADER
03:07:18.717 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.188.254:39373
03:07:18.719 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:07:18.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:18.719444 29427 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 29772
03:07:18.749 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.28.188.254:39373(127.28.188.254:39373)] handling channelInactive
03:07:18.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.188.254:39373(127.28.188.254:39373)] cleaning up while in state READY due to: connection closed
03:07:18.750 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:18.750 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.188.252:43955
03:07:18.750 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:07:18.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:18.751003 29427 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 29844
03:07:18.783 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:18.784 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.188.253:44093
03:07:18.784 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:07:18.784 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.28.188.252:43955(127.28.188.252:43955)] handling channelInactive
03:07:18.784 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.188.252:43955(127.28.188.252:43955)] cleaning up while in state READY due to: connection closed
03:07:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:18.784865 29427 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 29939
03:07:18.814 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:18.814 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.28.188.253:44093(127.28.188.253:44093)] handling channelInactive
03:07:18.814 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.28.188.253:44093(127.28.188.253:44093)] cleaning up while in state READY due to: connection closed
03:07:18.814 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.188.254:39373
03:07:18.814 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:18.815950 29427 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/logs
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/info.pb
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.254:39373
03:07:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.254
03:07:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35491
03:07:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.254
03:07:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:19.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:19.150507 30087 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:19.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:19.151314 30087 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:19.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:19.152235 30087 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:19.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:19.187453 30087 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:19.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:19.187750 30087 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:19.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:19.188203 30087 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:19.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:19.188529 30087 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:19.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:19.188827 30087 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:19.189127 30087 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:19.189379 30087 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41463/jwks.json
03:07:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:19.229079 30087 master_runner.cc:386] Master server non-default flags:
03:07:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data
03:07:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.254
03:07:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.254:39373
03:07:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/info.pb
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.254
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35491
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/logs
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:07:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:19.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:19.230811 30087 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:19.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:19.231916 30087 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:19.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:19.244412 30094 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:20.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:19.245321 30095 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:20.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:20.647532 30093 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 30087
03:07:20.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:20.917321 30087 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.673s user 0.529s sys 1.071s
03:07:20.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:20.918733 30087 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.674s user 0.530s sys 1.071s
03:07:20.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:20.921200 30097 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:20.924042 30096 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1673 milliseconds
03:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:20.924054 30087 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:20.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:20.925598 30087 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:20.928267 30087 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:20.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:20.929673 30087 hybrid_clock.cc:648] HybridClock initialized: now 1773976040929625 us; error 49 us; skew 500 ppm
03:07:20.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:20.930603 30087 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:20.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:20 dist-test-slave-sx75 krb5kdc[29438](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976040, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:20.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:20.936967 30087 init.cc:377] Logged in from keytab as kudu/127.28.188.254@KRBTEST.COM (short username kudu)
03:07:20.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:20.942667 30087 webserver.cc:492] Webserver started at http://127.28.188.254:35491/ using document root <none> and password file <none>
03:07:20.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:20.943643 30087 fs_manager.cc:362] Metadata directory not provided
03:07:20.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:20.943843 30087 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:20.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:20.952273 30087 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.002s
03:07:20.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:20.957837 30106 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:20.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:20.959168 30087 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
03:07:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:20.959599 30087 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4854c286195847a89cac7db7426b1b6e"
03:07:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:05 on dist-test-slave-sx75"
03:07:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:20.962333 30087 fs_report.cc:389] FS layout report
03:07:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/data
03:07:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.024712 30087 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:21.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.030679 30087 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:21.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.031165 30087 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:21.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.107002 30087 rpc_server.cc:307] RPC server started. Bound to: 127.28.188.254:39373
03:07:21.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.107093 30158 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.188.254:39373 every 8 connection(s)
03:07:21.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.109640 30087 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/info.pb
03:07:21.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.112808 29427 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 30087
03:07:21.113 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:21.113 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.188.252:43955
03:07:21.113 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.115711 29427 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/logs
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/info.pb
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.252:43955
03:07:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.252
03:07:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37157
03:07:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.252
03:07:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:21.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.124552 30159 sys_catalog.cc:263] Verifying existing consensus state
03:07:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.129848 30159 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Bootstrap starting.
03:07:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.169896 30159 log.cc:826] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Log is configured to *not* fsync() on all Append() calls
03:07:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.188817 30159 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:07:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.189684 30159 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Bootstrap complete.
03:07:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.213518 30159 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.215935 30159 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4854c286195847a89cac7db7426b1b6e, State: Initialized, Role: FOLLOWER
03:07:21.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.216921 30159 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:21.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.219966 30164 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:21.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.220782 30164 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: This master's current role is: FOLLOWER
03:07:21.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.222326 30159 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: configured and running, proceeding with master startup.
03:07:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.262048 30175 catalog_manager.cc:1269] Loaded cluster ID: 7b1010dc34f4435aa7dbf3461c94b984
03:07:21.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.262650 30175 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: loading cluster ID for follower catalog manager: success
03:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.271921 30175 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: acquiring CA information for follower catalog manager: success
03:07:21.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.279063 30175 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:21 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976040, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.254@KRBTEST.COM for kudu/127.28.188.254@KRBTEST.COM
03:07:21.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.315361 30087 master_runner.cc:186] Error getting master registration for 127.28.188.253:44093: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111),
03:07:21.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.343436 30087 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:21.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:21.493417 30161 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:21.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:21.494172 30161 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:21.495306 30161 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:21.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:21.544771 30161 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:21.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:21.545006 30161 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:21.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:21.545274 30161 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:21.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:21.545483 30161 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:21.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:21.545718 30161 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:21.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:21.545962 30161 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:21.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:21.546118 30161 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41463/jwks.json
03:07:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.590225 30161 master_runner.cc:386] Master server non-default flags:
03:07:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data
03:07:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.252
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.252:43955
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/info.pb
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.252
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37157
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/logs
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:21.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:21.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:21.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:07:21.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:21.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:07:21.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:07:21.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:21.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.591921 30161 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:21.593078 30161 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:21.604996 30184 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:22.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:22.717271 30189 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:07:22.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:22.718740 30189 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:22.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:22.738672 30189 leader_election.cc:290] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093), 58b878598d2e431ba4a31b17f243118d (127.28.188.252:43955)
03:07:22.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:22.761565 30108 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:22.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:22.773313 30108 leader_election.cc:336] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093): Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:22.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:22.779068 30108 leader_election.cc:336] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 58b878598d2e431ba4a31b17f243118d (127.28.188.252:43955): Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:22.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:22.780128 30108 leader_election.cc:304] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4854c286195847a89cac7db7426b1b6e; no voters: 58b878598d2e431ba4a31b17f243118d, 9362c8297aa946dfbcd513a95fa97f51
03:07:22.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:22.781137 30189 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
03:07:23.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:23.007660 30183 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 30161
03:07:23.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:23.036620 30161 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.433s user 0.558s sys 0.863s
03:07:23.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:23.037002 30161 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.433s user 0.558s sys 0.863s
03:07:23.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:21.606315 30185 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:23.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:23.051347 30193 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:23.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.051362 30161 server_base.cc:1061] running on GCE node
03:07:23.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.052937 30161 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:23.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.055845 30161 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:23.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.057293 30161 hybrid_clock.cc:648] HybridClock initialized: now 1773976043057189 us; error 108 us; skew 500 ppm
03:07:23.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.058398 30161 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:23 dist-test-slave-sx75 krb5kdc[29438](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976043, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:23.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.065956 30161 init.cc:377] Logged in from keytab as kudu/127.28.188.252@KRBTEST.COM (short username kudu)
03:07:23.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.071768 30161 webserver.cc:492] Webserver started at http://127.28.188.252:37157/ using document root <none> and password file <none>
03:07:23.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.072984 30161 fs_manager.cc:362] Metadata directory not provided
03:07:23.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.073263 30161 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:23.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.084148 30161 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.002s
03:07:23.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.089654 30201 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:23.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.090976 30161 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
03:07:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.091375 30161 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58b878598d2e431ba4a31b17f243118d"
03:07:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:09 on dist-test-slave-sx75"
03:07:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.094024 30161 fs_report.cc:389] FS layout report
03:07:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:23.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/data
03:07:23.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:23.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:23.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:23.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:23.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:23.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:23.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:23.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:23.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:23.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:23.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:23.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.161028 30161 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.171680 30161 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.172209 30161 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:23.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.265013 30161 rpc_server.cc:307] RPC server started. Bound to: 127.28.188.252:43955
03:07:23.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.265161 30253 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.188.252:43955 every 8 connection(s)
03:07:23.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.267998 30161 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/info.pb
03:07:23.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.277082 29427 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 30161
03:07:23.277 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:23.277 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.188.253:44093
03:07:23.278 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:07:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.279371 29427 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data
03:07:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/logs
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/info.pb
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.253:44093
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.253
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36353
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.253
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.284047 30254 sys_catalog.cc:263] Verifying existing consensus state
03:07:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.292008 30254 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Bootstrap starting.
03:07:23.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.344988 30254 log.cc:826] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Log is configured to *not* fsync() on all Append() calls
03:07:23.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.368415 30254 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:07:23.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.369597 30254 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Bootstrap complete.
03:07:23.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.409772 30254 raft_consensus.cc:359] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:23.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.413483 30254 raft_consensus.cc:740] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58b878598d2e431ba4a31b17f243118d, State: Initialized, Role: FOLLOWER
03:07:23.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.414745 30254 consensus_queue.cc:260] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.431710 30254 sys_catalog.cc:565] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: configured and running, proceeding with master startup.
03:07:23.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.431264 30258 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:23.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.435811 30258 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: This master's current role is: FOLLOWER
03:07:23.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.498452 30269 catalog_manager.cc:1269] Loaded cluster ID: 7b1010dc34f4435aa7dbf3461c94b984
03:07:23.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.502988 30269 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: loading cluster ID for follower catalog manager: success
03:07:23.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:23 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976043, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.252@KRBTEST.COM for kudu/127.28.188.254@KRBTEST.COM
03:07:23.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.525203 30269 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: acquiring CA information for follower catalog manager: success
03:07:23.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.538134 30269 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:23.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.549853 30161 master_runner.cc:186] Error getting master registration for 127.28.188.253:44093: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111),
03:07:23.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.582401 30161 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:23.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:23.746404 30256 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:23.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:23.747128 30256 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:23.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:23.748205 30256 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:23.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:23.784494 30256 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:23.784793 30256 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:23.785228 30256 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:23.785566 30256 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:23.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:23.785923 30256 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:23.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:23.786298 30256 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:23.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:23.786549 30256 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41463/jwks.json
03:07:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.829203 30256 master_runner.cc:386] Master server non-default flags:
03:07:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data
03:07:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.253
03:07:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.253:44093
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/info.pb
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.253
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36353
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/logs
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:23.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.830978 30256 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:23.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:23.832139 30256 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:23.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:23.848407 30278 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:24.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:24.285871 30250 messenger.cc:194] Invalid argument: socket address is ambiguous: no data on pending connections for acceptor pool at 127.28.188.252:43955
03:07:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.323323 30285 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:07:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.324645 30285 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:24.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.364076 30285 leader_election.cc:290] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093), 58b878598d2e431ba4a31b17f243118d (127.28.188.252:43955)
03:07:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:24.406396 30108 leader_election.cc:336] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093): Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:24.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.448678 30228 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4854c286195847a89cac7db7426b1b6e" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "58b878598d2e431ba4a31b17f243118d" is_pre_election: true
03:07:24.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.449608 30228 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4854c286195847a89cac7db7426b1b6e in term 2.
03:07:24.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.453258 30108 leader_election.cc:304] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4854c286195847a89cac7db7426b1b6e, 58b878598d2e431ba4a31b17f243118d; no voters: 9362c8297aa946dfbcd513a95fa97f51
03:07:24.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.456755 30285 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 2 FOLLOWER]: Leader pre-election won for term 3
03:07:24.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.457113 30285 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:07:24.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.457404 30285 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 2 FOLLOWER]: Advancing to term 3
03:07:24.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.474332 30285 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:24.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.487210 30285 leader_election.cc:290] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 3 election: Requested vote from peers 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093), 58b878598d2e431ba4a31b17f243118d (127.28.188.252:43955)
03:07:24.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.495121 30228 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4854c286195847a89cac7db7426b1b6e" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "58b878598d2e431ba4a31b17f243118d"
03:07:24.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.495875 30228 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 2 FOLLOWER]: Advancing to term 3
03:07:24.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:24.510890 30108 leader_election.cc:336] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093): Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:24.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.540804 30228 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4854c286195847a89cac7db7426b1b6e in term 3.
03:07:24.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.557003 30108 leader_election.cc:304] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4854c286195847a89cac7db7426b1b6e, 58b878598d2e431ba4a31b17f243118d; no voters: 9362c8297aa946dfbcd513a95fa97f51
03:07:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.558771 30285 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 3 FOLLOWER]: Leader election won for term 3
03:07:24.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.583884 30285 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 3 LEADER]: Becoming Leader. State: Replica: 4854c286195847a89cac7db7426b1b6e, State: Running, Role: LEADER
03:07:24.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.585343 30285 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:24.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.616197 30291 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4854c286195847a89cac7db7426b1b6e. Latest consensus state: current_term: 3 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:24.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.624773 30291 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: This master's current role is: LEADER
03:07:24.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.636700 30293 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:07:24.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.666913 30293 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:07:24.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.680192 30293 catalog_manager.cc:1269] Loaded cluster ID: 7b1010dc34f4435aa7dbf3461c94b984
03:07:24.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.680662 30293 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:07:24.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.700486 30293 catalog_manager.cc:1514] Loading token signing keys...
03:07:24.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.708904 30293 catalog_manager.cc:6055] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Loaded TSK: 0
03:07:24.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:24.710873 30293 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:07:25.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.113238 30228 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 3 FOLLOWER]: Refusing update from remote peer 4854c286195847a89cac7db7426b1b6e: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
03:07:25.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.128863 30291 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [LEADER]: Connected to new peer: Peer: permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
03:07:25.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:25.148206 30108 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e -> Peer 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093): Couldn't send request to peer 9362c8297aa946dfbcd513a95fa97f51. Status: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:07:25.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.154021 30296 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4854c286195847a89cac7db7426b1b6e. Latest consensus state: current_term: 3 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:25.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.155085 30296 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: This master's current role is: FOLLOWER
03:07:25.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.191162 30291 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:25.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.204936 30296 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:25.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.206198 30296 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: This master's current role is: FOLLOWER
03:07:25.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.208747 30291 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: This master's current role is: LEADER
03:07:25.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:25.249374 30277 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 30256
03:07:25.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:25.677500 30277 kernel_stack_watchdog.cc:198] Thread 30256 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 395ms:
03:07:25.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:07:25.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:07:25.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:07:25.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:07:25.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:23.851303 30279 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:25.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:25.680537 30256 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.829s user 0.414s sys 0.757s
03:07:25.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:25.680967 30256 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.830s user 0.414s sys 0.757s
03:07:25.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:25.681234 30281 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:25.683423 30280 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1831 milliseconds
03:07:25.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.683431 30256 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:25.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.685029 30256 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:25.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.688220 30256 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.690639 30256 hybrid_clock.cc:648] HybridClock initialized: now 1773976045690582 us; error 64 us; skew 500 ppm
03:07:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.691865 30256 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:25 dist-test-slave-sx75 krb5kdc[29438](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976045, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:25.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.699767 30256 init.cc:377] Logged in from keytab as kudu/127.28.188.253@KRBTEST.COM (short username kudu)
03:07:25.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.707346 30256 webserver.cc:492] Webserver started at http://127.28.188.253:36353/ using document root <none> and password file <none>
03:07:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.708639 30256 fs_manager.cc:362] Metadata directory not provided
03:07:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.708918 30256 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:25.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.718926 30256 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.004s sys 0.003s
03:07:25.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.724193 30308 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:25.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.725432 30256 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
03:07:25.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.725739 30256 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:25.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9362c8297aa946dfbcd513a95fa97f51"
03:07:25.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:07 on dist-test-slave-sx75"
03:07:25.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.727711 30256 fs_report.cc:389] FS layout report
03:07:25.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:25.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:25.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:25.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/data
03:07:25.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:25.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:25.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:25.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:25.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:25.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:25.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:25.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:25.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:25.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:25.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.792438 30256 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.798776 30256 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.799209 30256 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.895712 30256 rpc_server.cc:307] RPC server started. Bound to: 127.28.188.253:44093
03:07:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.895774 30361 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.188.253:44093 every 8 connection(s)
03:07:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.898581 30256 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/info.pb
03:07:25.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.906257 29427 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 30256
03:07:25.906 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:25.907 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.188.254:39373(127.28.188.254:39373)] connecting to peer
03:07:25.909 [DEBUG - kudu-nio-3] (Connection.java:757) [peer master-127.28.188.254:39373(127.28.188.254:39373)] Successfully connected to peer
03:07:25.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.913419 30362 sys_catalog.cc:263] Verifying existing consensus state
03:07:25.914 [DEBUG - kudu-nio-3] (SecurityContext.java:222) Refreshing Kerberos credentials...
03:07:25.915 [DEBUG - kudu-nio-3] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc
03:07:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:25 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773976029, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.921264 30362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Bootstrap starting.
03:07:25.921 [DEBUG - kudu-nio-3] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:07:25.922 [INFO - kudu-nio-3] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
03:07:25.923 [DEBUG - kudu-nio-3] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.28.188.254
03:07:25.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:25 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773976029, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.188.254@KRBTEST.COM
03:07:25.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:25.975559 30362 log.cc:826] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Log is configured to *not* fsync() on all Append() calls
03:07:25.979 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0xe06acd7f, L:/127.0.0.1:46162 - R:/127.28.188.254:39373] using SASL/GSSAPI
03:07:25.982 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:07:25.982 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:07:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.002038 30362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:07:26.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.003122 30362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Bootstrap complete.
03:07:26.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:26 dist-test-slave-sx75 krb5kdc[29438](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976046, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:26.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:07:26.016 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:26.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.039026 30362 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:26.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.042394 30362 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9362c8297aa946dfbcd513a95fa97f51, State: Initialized, Role: FOLLOWER
03:07:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.043435 30362 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.046401 30369 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.047403 30369 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: This master's current role is: FOLLOWER
03:07:26.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.049170 30362 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: configured and running, proceeding with master startup.
03:07:26.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:26 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976040, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.254@KRBTEST.COM for kudu/127.28.188.253@KRBTEST.COM
03:07:26.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.091760 30337 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 2 FOLLOWER]: Advancing to term 3
03:07:26.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:26 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976045, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.253@KRBTEST.COM for kudu/127.28.188.254@KRBTEST.COM
03:07:26.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.148641 30369 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4854c286195847a89cac7db7426b1b6e. Latest consensus state: current_term: 3 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:26.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.149345 30369 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: This master's current role is: FOLLOWER
03:07:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:26 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976045, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.253@KRBTEST.COM for kudu/127.28.188.253@KRBTEST.COM
03:07:26.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.151880 30386 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:26.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.152611 30386 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: This master's current role is: FOLLOWER
03:07:26.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.163477 30382 catalog_manager.cc:1269] Loaded cluster ID: 7b1010dc34f4435aa7dbf3461c94b984
03:07:26.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.164042 30382 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: loading cluster ID for follower catalog manager: success
03:07:26.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.182489 30382 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: acquiring CA information for follower catalog manager: success
03:07:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.189621 30382 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:26.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:26 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976045, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.253@KRBTEST.COM for kudu/127.28.188.252@KRBTEST.COM
03:07:26.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.621443 30291 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:26.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:26.622218 30291 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: This master's current role is: LEADER
03:07:27.017 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.188.254:39373
03:07:27.018 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:07:27.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:27.018435 29427 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 30087
03:07:27.060 [DEBUG - kudu-nio-3] (Connection.java:239) [peer master-127.28.188.254:39373(127.28.188.254:39373)] handling channelInactive
03:07:27.060 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.28.188.254:39373(127.28.188.254:39373)] cleaning up while in state READY due to: connection closed
03:07:27.061 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:27.061 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.188.252:43955
03:07:27.061 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:07:27.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:27.061981 29427 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 30161
03:07:27.093 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:27.093 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.188.253:44093
03:07:27.094 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:07:27.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:27.094349 29427 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 30256
03:07:27.120 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:27.120 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.188.254:39373
03:07:27.121 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:27.122320 29427 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data
03:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/logs
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/info.pb
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.254:39373
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.254
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35491
03:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.254
03:07:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:27.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:27.448774 30396 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:27.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:27.449378 30396 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:27.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:27.450121 30396 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:27.485687 30396 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:27.486006 30396 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:27.486300 30396 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:27.486508 30396 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:27.486735 30396 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:27.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:27.486933 30396 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:27.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:27.487082 30396 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41463/jwks.json
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:27.547689 30396 master_runner.cc:386] Master server non-default flags:
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.254
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.254:39373
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/info.pb
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.254
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35491
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/logs
03:07:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:07:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:07:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:07:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:27.549435 30396 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:27.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:27.550592 30396 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:27.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:27.561388 30402 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:29.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:28.964964 30401 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 30396
03:07:29.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:29.326289 30401 kernel_stack_watchdog.cc:198] Thread 30396 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
03:07:29.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:07:29.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:07:29.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:07:29.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:07:29.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:27.563198 30403 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:29.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:29.327878 30404 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1763 milliseconds
03:07:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:29.333036 30396 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.770s user 0.003s sys 0.004s
03:07:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:29.333451 30396 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.771s user 0.003s sys 0.004s
03:07:29.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:29.335186 30405 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:29.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.335245 30396 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:29.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.336567 30396 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:29.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.339128 30396 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:29.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.340543 30396 hybrid_clock.cc:648] HybridClock initialized: now 1773976049340495 us; error 50 us; skew 500 ppm
03:07:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.341435 30396 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:29.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:29 dist-test-slave-sx75 krb5kdc[29438](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976049, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:29.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.347687 30396 init.cc:377] Logged in from keytab as kudu/127.28.188.254@KRBTEST.COM (short username kudu)
03:07:29.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.353368 30396 webserver.cc:492] Webserver started at http://127.28.188.254:35491/ using document root <none> and password file <none>
03:07:29.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.354250 30396 fs_manager.cc:362] Metadata directory not provided
03:07:29.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.354454 30396 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:29.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.362602 30396 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.003s sys 0.004s
03:07:29.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.368409 30415 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:29.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.369575 30396 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
03:07:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.369949 30396 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4854c286195847a89cac7db7426b1b6e"
03:07:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:05 on dist-test-slave-sx75"
03:07:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.372030 30396 fs_report.cc:389] FS layout report
03:07:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/data
03:07:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.425325 30396 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:29.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.434319 30396 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:29.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.434907 30396 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.517247 30396 rpc_server.cc:307] RPC server started. Bound to: 127.28.188.254:39373
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.517364 30467 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.188.254:39373 every 8 connection(s)
03:07:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.521404 30396 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/info.pb
03:07:29.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.524873 29427 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 30396
03:07:29.525 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:29.525 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.188.252:43955
03:07:29.525 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:07:29.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.527455 29427 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:29.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:29.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/logs
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/info.pb
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.252:43955
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.252
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37157
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.252
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.541491 30468 sys_catalog.cc:263] Verifying existing consensus state
03:07:29.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.547125 30468 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Bootstrap starting.
03:07:29.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.585471 30468 log.cc:826] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Log is configured to *not* fsync() on all Append() calls
03:07:29.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.603475 30468 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:07:29.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.604475 30468 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Bootstrap complete.
03:07:29.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.628196 30468 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:29.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.630625 30468 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4854c286195847a89cac7db7426b1b6e, State: Initialized, Role: FOLLOWER
03:07:29.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.631525 30468 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:29.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.634486 30472 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:29.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.635386 30472 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: This master's current role is: FOLLOWER
03:07:29.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.636365 30468 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: configured and running, proceeding with master startup.
03:07:29.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.667609 30483 catalog_manager.cc:1269] Loaded cluster ID: 7b1010dc34f4435aa7dbf3461c94b984
03:07:29.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.668145 30483 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: loading cluster ID for follower catalog manager: success
03:07:29.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.681767 30483 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: acquiring CA information for follower catalog manager: success
03:07:29.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.689020 30483 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:29.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:29 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976049, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.254@KRBTEST.COM for kudu/127.28.188.254@KRBTEST.COM
03:07:29.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.723644 30396 master_runner.cc:186] Error getting master registration for 127.28.188.253:44093: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111),
03:07:29.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.750442 30396 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:29.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:29.881228 30470 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:29.882004 30470 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:29.882934 30470 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:29.916885 30470 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:29.917178 30470 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:29.917472 30470 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:29.917690 30470 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:29.917887 30470 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:29.918120 30470 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:29.918313 30470 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41463/jwks.json
03:07:29.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.960142 30470 master_runner.cc:386] Master server non-default flags:
03:07:29.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:29.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:29.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:29.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:29.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data
03:07:29.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:29.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:29.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:29.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.252
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.252:43955
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/info.pb
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.252
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37157
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:29.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:29.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:29.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:29.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/logs
03:07:29.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:29.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:29.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:29.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:29.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:29.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:07:29.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:29.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:07:29.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:07:29.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:29.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.961783 30470 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:29.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:29.963227 30470 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:29.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:29.979128 30493 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:30.986904 30500 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:07:30.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:30.987495 30500 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:30.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:30.990490 30500 leader_election.cc:290] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093), 58b878598d2e431ba4a31b17f243118d (127.28.188.252:43955)
03:07:30.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:30.996435 30417 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:31.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:31.004591 30417 leader_election.cc:336] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093): Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:31.005488 30417 leader_election.cc:336] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 58b878598d2e431ba4a31b17f243118d (127.28.188.252:43955): Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.006057 30417 leader_election.cc:304] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4854c286195847a89cac7db7426b1b6e; no voters: 58b878598d2e431ba4a31b17f243118d, 9362c8297aa946dfbcd513a95fa97f51
03:07:31.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.007154 30500 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
03:07:31.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:29.984575 30494 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:29.983027 30496 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:31.256040 30495 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
03:07:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.256148 30470 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:31.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.260990 30470 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:31.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.264074 30470 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.265535 30470 hybrid_clock.cc:648] HybridClock initialized: now 1773976051265471 us; error 75 us; skew 500 ppm
03:07:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.266340 30470 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:31.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:31 dist-test-slave-sx75 krb5kdc[29438](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976051, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:31.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.272425 30470 init.cc:377] Logged in from keytab as kudu/127.28.188.252@KRBTEST.COM (short username kudu)
03:07:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.278827 30470 webserver.cc:492] Webserver started at http://127.28.188.252:37157/ using document root <none> and password file <none>
03:07:31.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.279937 30470 fs_manager.cc:362] Metadata directory not provided
03:07:31.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.280186 30470 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:31.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.289261 30470 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
03:07:31.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.294826 30509 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:31.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.297159 30470 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.003s sys 0.000s
03:07:31.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.297500 30470 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:31.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58b878598d2e431ba4a31b17f243118d"
03:07:31.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:09 on dist-test-slave-sx75"
03:07:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.299549 30470 fs_report.cc:389] FS layout report
03:07:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/data
03:07:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:31.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.363621 30470 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.369966 30470 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.370790 30470 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.460961 30470 rpc_server.cc:307] RPC server started. Bound to: 127.28.188.252:43955
03:07:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.461117 30561 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.188.252:43955 every 8 connection(s)
03:07:31.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.463838 30470 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/info.pb
03:07:31.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.464460 29427 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 30470
03:07:31.465 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:31.465 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.188.253:44093
03:07:31.465 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:07:31.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.467278 29427 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:31.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:31.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:31.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data
03:07:31.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:31.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/logs
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/info.pb
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.253:44093
03:07:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.253
03:07:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36353
03:07:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.253
03:07:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.485523 30562 sys_catalog.cc:263] Verifying existing consensus state
03:07:31.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.494455 30562 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Bootstrap starting.
03:07:31.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.550626 30562 log.cc:826] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Log is configured to *not* fsync() on all Append() calls
03:07:31.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.603040 30562 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:07:31.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.603935 30562 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Bootstrap complete.
03:07:31.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.628018 30562 raft_consensus.cc:359] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:31.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.630447 30562 raft_consensus.cc:740] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58b878598d2e431ba4a31b17f243118d, State: Initialized, Role: FOLLOWER
03:07:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.631460 30562 consensus_queue.cc:260] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:31.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.634511 30566 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:31.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.636255 30566 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: This master's current role is: FOLLOWER
03:07:31.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.636724 30562 sys_catalog.cc:565] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: configured and running, proceeding with master startup.
03:07:31.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.696705 30577 catalog_manager.cc:1269] Loaded cluster ID: 7b1010dc34f4435aa7dbf3461c94b984
03:07:31.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.697013 30577 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: loading cluster ID for follower catalog manager: success
03:07:31.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.706017 30577 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: acquiring CA information for follower catalog manager: success
03:07:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.711995 30577 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:31.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:31 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976051, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.252@KRBTEST.COM for kudu/127.28.188.254@KRBTEST.COM
03:07:31.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.746021 30470 master_runner.cc:186] Error getting master registration for 127.28.188.253:44093: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111),
03:07:31.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:31.774981 30470 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:31.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:31.929035 30564 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:31.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:31.929816 30564 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:31.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:31.930585 30564 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:31.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:31.972679 30564 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:31.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:31.972963 30564 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:31.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:31.973400 30564 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:31.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:31.973731 30564 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:31.974031 30564 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:31.974339 30564 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:31.974582 30564 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41463/jwks.json
03:07:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:32.019474 30564 master_runner.cc:386] Master server non-default flags:
03:07:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.253
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.253:44093
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/info.pb
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.253
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36353
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/logs
03:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:32.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:32.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:32.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:32.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:32.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:07:32.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:32.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:07:32.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:07:32.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:32.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:32.021111 30564 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:32.022209 30564 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:32.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:32.033950 30587 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:32.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:32.504148 30558 messenger.cc:194] Invalid argument: socket address is ambiguous: no data on pending connections for acceptor pool at 127.28.188.252:43955
03:07:32.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:32.969817 30592 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:07:32.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:32.974033 30592 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:32.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:32.992533 30592 leader_election.cc:290] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093), 58b878598d2e431ba4a31b17f243118d (127.28.188.252:43955)
03:07:33.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:33.025475 30417 leader_election.cc:336] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093): Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:33.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.039952 30537 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4854c286195847a89cac7db7426b1b6e" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "58b878598d2e431ba4a31b17f243118d" is_pre_election: true
03:07:33.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.041028 30537 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4854c286195847a89cac7db7426b1b6e in term 3.
03:07:33.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.042882 30417 leader_election.cc:304] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4854c286195847a89cac7db7426b1b6e, 58b878598d2e431ba4a31b17f243118d; no voters: 9362c8297aa946dfbcd513a95fa97f51
03:07:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.044934 30592 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 3 FOLLOWER]: Leader pre-election won for term 4
03:07:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.045323 30592 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:07:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.045706 30592 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 3 FOLLOWER]: Advancing to term 4
03:07:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.057824 30592 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:33.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.064009 30537 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4854c286195847a89cac7db7426b1b6e" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "58b878598d2e431ba4a31b17f243118d"
03:07:33.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.064626 30537 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 3 FOLLOWER]: Advancing to term 4
03:07:33.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:33.075042 30417 leader_election.cc:336] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093): Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:33.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.080240 30592 leader_election.cc:290] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 4 election: Requested vote from peers 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093), 58b878598d2e431ba4a31b17f243118d (127.28.188.252:43955)
03:07:33.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.084825 30537 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4854c286195847a89cac7db7426b1b6e in term 4.
03:07:33.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.086169 30417 leader_election.cc:304] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4854c286195847a89cac7db7426b1b6e, 58b878598d2e431ba4a31b17f243118d; no voters: 9362c8297aa946dfbcd513a95fa97f51
03:07:33.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.087471 30592 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 4 FOLLOWER]: Leader election won for term 4
03:07:33.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.095314 30592 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 4 LEADER]: Becoming Leader. State: Replica: 4854c286195847a89cac7db7426b1b6e, State: Running, Role: LEADER
03:07:33.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.096379 30592 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:33.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.115716 30597 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4854c286195847a89cac7db7426b1b6e. Latest consensus state: current_term: 4 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:33.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.116783 30597 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: This master's current role is: LEADER
03:07:33.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.128033 30599 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:07:33.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.139051 30599 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:07:33.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.144768 30599 catalog_manager.cc:1269] Loaded cluster ID: 7b1010dc34f4435aa7dbf3461c94b984
03:07:33.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.145166 30599 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:07:33.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.151294 30599 catalog_manager.cc:1514] Loading token signing keys...
03:07:33.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.156908 30599 catalog_manager.cc:6055] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Loaded TSK: 0
03:07:33.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.158639 30599 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:07:33.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:33.551447 30417 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e -> Peer 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093): Couldn't send request to peer 9362c8297aa946dfbcd513a95fa97f51. Status: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:07:33.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:32.036401 30588 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:33.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:33.435508 30586 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 30564
03:07:33.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:33.587634 30564 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.553s user 0.513s sys 0.859s
03:07:33.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:33.588979 30564 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.554s user 0.513s sys 0.859s
03:07:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:33.598037 30601 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.598486 30564 server_base.cc:1061] running on GCE node
03:07:33.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.599864 30564 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.602895 30564 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.604332 30564 hybrid_clock.cc:648] HybridClock initialized: now 1773976053604248 us; error 85 us; skew 500 ppm
03:07:33.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.605221 30564 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:33.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:33 dist-test-slave-sx75 krb5kdc[29438](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976053, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:33.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.612257 30564 init.cc:377] Logged in from keytab as kudu/127.28.188.253@KRBTEST.COM (short username kudu)
03:07:33.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.613360 30537 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 4 FOLLOWER]: Refusing update from remote peer 4854c286195847a89cac7db7426b1b6e: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
03:07:33.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.615598 30597 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [LEADER]: Connected to new peer: Peer: permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
03:07:33.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.619570 30564 webserver.cc:492] Webserver started at http://127.28.188.253:36353/ using document root <none> and password file <none>
03:07:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.620818 30564 fs_manager.cc:362] Metadata directory not provided
03:07:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.621099 30564 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:33.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.633109 30606 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4854c286195847a89cac7db7426b1b6e. Latest consensus state: current_term: 4 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:33.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.634034 30606 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: This master's current role is: FOLLOWER
03:07:33.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.645162 30564 fs_manager.cc:714] Time spent opening directory manager: real 0.020s user 0.019s sys 0.001s
03:07:33.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.650085 30606 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:33.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.650854 30606 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: This master's current role is: FOLLOWER
03:07:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.650523 30609 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.652707 30609 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: This master's current role is: LEADER
03:07:33.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.654044 30613 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:33.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.655493 30564 fs_manager.cc:730] Time spent opening block manager: real 0.008s user 0.003s sys 0.001s
03:07:33.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.655900 30564 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:33.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9362c8297aa946dfbcd513a95fa97f51"
03:07:33.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:07 on dist-test-slave-sx75"
03:07:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.658664 30564 fs_report.cc:389] FS layout report
03:07:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/data
03:07:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:33.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.724785 30564 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.732308 30564 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:33.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.732873 30564 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:33.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.824373 30564 rpc_server.cc:307] RPC server started. Bound to: 127.28.188.253:44093
03:07:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.824540 30665 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.188.253:44093 every 8 connection(s)
03:07:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.828333 30564 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/info.pb
03:07:33.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.836658 29427 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 30564
03:07:33.837 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:33.838 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.188.254:39373(127.28.188.254:39373)] connecting to peer
03:07:33.840 [DEBUG - kudu-nio-4] (Connection.java:757) [peer master-127.28.188.254:39373(127.28.188.254:39373)] Successfully connected to peer
03:07:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.843005 30666 sys_catalog.cc:263] Verifying existing consensus state
03:07:33.857 [DEBUG - kudu-nio-4] (SecurityContext.java:222) Refreshing Kerberos credentials...
03:07:33.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.857236 30666 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Bootstrap starting.
03:07:33.857 [DEBUG - kudu-nio-4] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc
03:07:33.862 [DEBUG - kudu-nio-4] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:07:33.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:33 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773976046, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:33.862 [INFO - kudu-nio-4] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
03:07:33.864 [DEBUG - kudu-nio-4] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.28.188.254
03:07:33.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:33 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773976046, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.188.254@KRBTEST.COM
03:07:33.903 [DEBUG - kudu-nio-4] (Negotiator.java:919) Authenticated connection [id: 0x9582e64e, L:/127.0.0.1:52932 - R:/127.28.188.254:39373] using SASL/GSSAPI
03:07:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.913800 30666 log.cc:826] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Log is configured to *not* fsync() on all Append() calls
03:07:33.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.937893 30666 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:07:33.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.939036 30666 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Bootstrap complete.
03:07:33.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.974262 30666 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:33.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.977710 30666 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9362c8297aa946dfbcd513a95fa97f51, State: Initialized, Role: FOLLOWER
03:07:33.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.978792 30666 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:33.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.981588 30671 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:33.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.982213 30671 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: This master's current role is: FOLLOWER
03:07:33.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:33.983618 30666 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: configured and running, proceeding with master startup.
03:07:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:34.025487 30682 catalog_manager.cc:1269] Loaded cluster ID: 7b1010dc34f4435aa7dbf3461c94b984
03:07:34.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:34.025743 30682 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: loading cluster ID for follower catalog manager: success
03:07:34.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:34 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976053, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.253@KRBTEST.COM for kudu/127.28.188.254@KRBTEST.COM
03:07:34.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:34.040618 30682 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: acquiring CA information for follower catalog manager: success
03:07:34.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:34.056124 30682 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:34.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:34.162305 30640 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [term 3 FOLLOWER]: Advancing to term 4
03:07:34.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:34.178308 30692 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:34.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:34.180827 30692 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: This master's current role is: FOLLOWER
03:07:34.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:34.183758 30671 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4854c286195847a89cac7db7426b1b6e. Latest consensus state: current_term: 4 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:34.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:34.185122 30671 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51 [sys.catalog]: This master's current role is: FOLLOWER
03:07:34.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:34.742436 30609 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:34.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:34.743233 30609 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: This master's current role is: LEADER
03:07:34.906 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.188.254:39373
03:07:34.907 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:07:34.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:34.907414 29427 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 30396
03:07:34.942 [DEBUG - kudu-nio-4] (Connection.java:239) [peer master-127.28.188.254:39373(127.28.188.254:39373)] handling channelInactive
03:07:34.943 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.28.188.254:39373(127.28.188.254:39373)] cleaning up while in state READY due to: connection closed
03:07:34.943 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:34.943 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.188.252:43955
03:07:34.943 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:07:34.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:34.943715 29427 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 30470
03:07:34.983 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:34.983 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.188.253:44093
03:07:34.984 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:07:34.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:34.984380 29427 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 30564
03:07:35.015 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:35.016 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.188.254:39373
03:07:35.016 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:07:35.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:35.017405 29427 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:35.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:35.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:35.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/logs
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/info.pb
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.254:39373
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.254
03:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35491
03:07:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.254
03:07:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:35.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:35.563546 30697 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:35.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:35.565210 30697 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:35.567526 30697 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:35.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:35.652788 30697 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:35.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:35.653553 30697 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:35.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:35.654413 30697 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:35.655179 30697 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:35.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:35.655870 30697 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:35.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:35.656739 30697 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:35.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:35.657297 30697 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41463/jwks.json
03:07:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:35.774027 30697 master_runner.cc:386] Master server non-default flags:
03:07:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data
03:07:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.254
03:07:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.254:39373
03:07:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/info.pb
03:07:35.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.254
03:07:35.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35491
03:07:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/logs
03:07:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:35.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:07:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:07:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:07:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:35.778448 30697 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:35.781968 30697 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:35.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:35.810285 30704 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:36.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:36.333990 29125 debug-util.cc:398] Leaking SignalData structure 0x7b08000b75e0 after lost signal to thread 28995
03:07:36.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:36.439046 29125 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9200 after lost signal to thread 29128
03:07:36.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:36.463836 29269 debug-util.cc:398] Leaking SignalData structure 0x7b08000b32c0 after lost signal to thread 29140
03:07:36.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:36.508556 29269 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8b60 after lost signal to thread 29272
03:07:40.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:37.126896 30703 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 30697
03:07:40.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:40.181115 30703 kernel_stack_watchdog.cc:198] Thread 30697 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 295ms:
03:07:40.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:07:40.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:07:40.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:07:40.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:40.183298 30697 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 4.360s user 0.927s sys 2.007s
03:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:40.183764 30697 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 4.361s user 0.928s sys 2.007s
03:07:40.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:35.821269 30705 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:40.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:40.184386 30707 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:40.189338 30706 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 4355 milliseconds
03:07:40.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.189934 30697 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:40.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.191934 30697 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:40.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.195438 30697 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:40.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.196892 30697 hybrid_clock.cc:648] HybridClock initialized: now 1773976060196806 us; error 87 us; skew 500 ppm
03:07:40.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.198148 30697 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:40.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:40 dist-test-slave-sx75 krb5kdc[29438](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976060, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.207630 30697 init.cc:377] Logged in from keytab as kudu/127.28.188.254@KRBTEST.COM (short username kudu)
03:07:40.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.214655 30697 webserver.cc:492] Webserver started at http://127.28.188.254:35491/ using document root <none> and password file <none>
03:07:40.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.216041 30697 fs_manager.cc:362] Metadata directory not provided
03:07:40.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.216351 30697 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:40.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.229691 30697 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.001s sys 0.007s
03:07:40.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.236539 30720 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.238094 30697 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.000s sys 0.004s
03:07:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.238569 30697 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4854c286195847a89cac7db7426b1b6e"
03:07:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:05 on dist-test-slave-sx75"
03:07:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.241725 30697 fs_report.cc:389] FS layout report
03:07:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/wal
03:07:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/data
03:07:40.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:40.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:40.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:40.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:40.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:40.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:40.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:40.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:40.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:40.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:40.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:40.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.317651 30697 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:40.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.329895 30697 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:40.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.330544 30697 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:40.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.498926 30697 rpc_server.cc:307] RPC server started. Bound to: 127.28.188.254:39373
03:07:40.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.499035 30772 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.188.254:39373 every 8 connection(s)
03:07:40.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.508350 30697 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-0/data/info.pb
03:07:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.509706 29427 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 30697
03:07:40.510 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:40.510 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.188.252:43955
03:07:40.510 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.512652 29427 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/logs
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/info.pb
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.252:43955
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.252
03:07:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37157
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.252
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:40.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.543053 30775 sys_catalog.cc:263] Verifying existing consensus state
03:07:40.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.551137 30775 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Bootstrap starting.
03:07:40.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.605746 30775 log.cc:826] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Log is configured to *not* fsync() on all Append() calls
03:07:40.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.637105 30775 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:07:40.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.638134 30775 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Bootstrap complete.
03:07:40.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:40.664844 29412 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8a80 after lost signal to thread 29285
03:07:40.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:40.666004 29412 debug-util.cc:398] Leaking SignalData structure 0x7b08000adfe0 after lost signal to thread 29415
03:07:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.664526 30775 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.668390 30775 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4854c286195847a89cac7db7426b1b6e, State: Initialized, Role: FOLLOWER
03:07:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.669697 30775 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 4.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:40.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.683811 30779 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 4 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:40.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.684782 30779 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: This master's current role is: FOLLOWER
03:07:40.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.685930 30775 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: configured and running, proceeding with master startup.
03:07:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.767975 30790 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:07:40.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:40 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976060, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.254@KRBTEST.COM for kudu/127.28.188.254@KRBTEST.COM
03:07:40.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.832616 30697 master_runner.cc:186] Error getting master registration for 127.28.188.253:44093: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111),
03:07:40.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:40.860213 30697 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:40.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:40.984138 30777 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:40.984788 30777 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:40.985875 30777 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:41.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:41.019474 30777 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:41.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:41.019702 30777 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:41.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:41.020115 30777 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:41.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:41.020519 30777 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:41.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:41.020787 30777 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:41.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:41.020987 30777 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:41.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:41.021135 30777 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41463/jwks.json
03:07:41.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:41.060110 30777 master_runner.cc:386] Master server non-default flags:
03:07:41.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:41.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:41.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:41.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.252
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.252:43955
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/info.pb
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.252
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37157
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/logs
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:41.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:41.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:41.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:41.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:07:41.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:41.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:07:41.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:07:41.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:41.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:41.061359 30777 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:41.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:41.062296 30777 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:41.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:41.075264 30798 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:41.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:41.791945 30790 catalog_manager.cc:1269] Loaded cluster ID: 7b1010dc34f4435aa7dbf3461c94b984
03:07:41.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:41.792258 30790 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: loading cluster ID for follower catalog manager: success
03:07:41.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:41.810173 30790 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: acquiring CA information for follower catalog manager: success
03:07:41.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:41.824714 30790 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:41.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:41.919539 30805 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:07:41.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:41.920197 30805 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:41.930079 30805 leader_election.cc:290] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093), 58b878598d2e431ba4a31b17f243118d (127.28.188.252:43955)
03:07:41.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:41.942138 30722 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:41.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:41.956707 30722 leader_election.cc:336] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093): Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:41.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:41.957700 30722 leader_election.cc:336] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 58b878598d2e431ba4a31b17f243118d (127.28.188.252:43955): Network error: Client connection negotiation failed: client connection to 127.28.188.252:43955: connect: Connection refused (error 111)
03:07:41.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:41.958276 30722 leader_election.cc:304] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4854c286195847a89cac7db7426b1b6e; no voters: 58b878598d2e431ba4a31b17f243118d, 9362c8297aa946dfbcd513a95fa97f51
03:07:41.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:41.959427 30805 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
03:07:42.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:41.077661 30799 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:42.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:41.080813 30801 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:42.748809 30800 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1673 milliseconds
03:07:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:42.748988 30777 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:42.750699 30777 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:42.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:42.754877 30777 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:42.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:42.756358 30777 hybrid_clock.cc:648] HybridClock initialized: now 1773976062756282 us; error 74 us; skew 500 ppm
03:07:42.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:42.757551 30777 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:42.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:42 dist-test-slave-sx75 krb5kdc[29438](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976062, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:42.765750 30777 init.cc:377] Logged in from keytab as kudu/127.28.188.252@KRBTEST.COM (short username kudu)
03:07:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:42.772239 30777 webserver.cc:492] Webserver started at http://127.28.188.252:37157/ using document root <none> and password file <none>
03:07:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:42.773514 30777 fs_manager.cc:362] Metadata directory not provided
03:07:42.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:42.773805 30777 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:42.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:42.785910 30777 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.001s sys 0.004s
03:07:42.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:42.808133 30815 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:42.811599 30777 fs_manager.cc:730] Time spent opening block manager: real 0.023s user 0.013s sys 0.008s
03:07:42.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:42.812155 30777 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:42.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58b878598d2e431ba4a31b17f243118d"
03:07:42.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:09 on dist-test-slave-sx75"
03:07:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:42.814476 30777 fs_report.cc:389] FS layout report
03:07:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/wal
03:07:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/data
03:07:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:42.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:42.880007 30777 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:42.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:42.886857 30777 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:42.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:42.887394 30777 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.012328 30777 rpc_server.cc:307] RPC server started. Bound to: 127.28.188.252:43955
03:07:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.012355 30867 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.188.252:43955 every 8 connection(s)
03:07:43.017 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:43.017 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.188.253:44093
03:07:43.018 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:07:43.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.015609 30777 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-2/data/info.pb
03:07:43.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.016548 29427 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 30777
03:07:43.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.020428 29427 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/logs
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/info.pb
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.253:44093
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.253
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36353
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.253
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:43.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:43.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:43.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:43.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:43.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:43.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:43.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:43.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:43.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:43.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:43.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.042232 30868 sys_catalog.cc:263] Verifying existing consensus state
03:07:43.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.051815 30868 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Bootstrap starting.
03:07:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.106160 30868 log.cc:826] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Log is configured to *not* fsync() on all Append() calls
03:07:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.136502 30868 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:07:43.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.137574 30868 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: Bootstrap complete.
03:07:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.175028 30868 raft_consensus.cc:359] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.178733 30868 raft_consensus.cc:740] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58b878598d2e431ba4a31b17f243118d, State: Initialized, Role: FOLLOWER
03:07:43.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.179811 30868 consensus_queue.cc:260] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 4.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:43.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.183281 30872 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 4 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:43.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.184231 30872 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: This master's current role is: FOLLOWER
03:07:43.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.185307 30868 sys_catalog.cc:565] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: configured and running, proceeding with master startup.
03:07:43.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.255501 30883 catalog_manager.cc:1269] Loaded cluster ID: 7b1010dc34f4435aa7dbf3461c94b984
03:07:43.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.255821 30883 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: loading cluster ID for follower catalog manager: success
03:07:43.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:43 dist-test-slave-sx75 krb5kdc[29438](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976062, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.252@KRBTEST.COM for kudu/127.28.188.254@KRBTEST.COM
03:07:43.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.277561 30883 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: acquiring CA information for follower catalog manager: success
03:07:43.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.291507 30883 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.302060 30777 master_runner.cc:186] Error getting master registration for 127.28.188.253:44093: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111),
03:07:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.328963 30777 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:43.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:43.568773 30869 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:43.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:43.570933 30869 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:43.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:43.572293 30869 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:43.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:43.643069 30869 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:43.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:43.643623 30869 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:43.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:43.644197 30869 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:43.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:43.644692 30869 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:43.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:43.645117 30869 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:43.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:43.645519 30869 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:43.645877 30869 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41463/jwks.json
03:07:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.718919 30869 master_runner.cc:386] Master server non-default flags:
03:07:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.188.212:41253
03:07:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data
03:07:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.188.254:39373,127.28.188.253:44093,127.28.188.252:43955
03:07:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/kudu.keytab
03:07:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.188.253
03:07:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.188.253:44093
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41463/jwks.json
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/info.pb
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.188.253
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36353
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/logs
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:07:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:43.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:07:43.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:07:43.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:43.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.720903 30869 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:43.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.722343 30869 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:43.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:43.736353 30892 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:43.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.755788 30895 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:07:43.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.756467 30895 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:43.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.765450 30895 leader_election.cc:290] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093), 58b878598d2e431ba4a31b17f243118d (127.28.188.252:43955)
03:07:43.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:43.796305 30722 leader_election.cc:336] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093): Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:43.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.819851 30842 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4854c286195847a89cac7db7426b1b6e" candidate_term: 5 candidate_status { last_received { term: 4 index: 7 } } ignore_live_leader: false dest_uuid: "58b878598d2e431ba4a31b17f243118d" is_pre_election: true
03:07:43.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.820807 30842 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4854c286195847a89cac7db7426b1b6e in term 4.
03:07:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.822495 30722 leader_election.cc:304] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4854c286195847a89cac7db7426b1b6e, 58b878598d2e431ba4a31b17f243118d; no voters: 9362c8297aa946dfbcd513a95fa97f51
03:07:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.823627 30895 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 4 FOLLOWER]: Leader pre-election won for term 5
03:07:43.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.823992 30895 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 4 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:07:43.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.824412 30895 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 4 FOLLOWER]: Advancing to term 5
03:07:43.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.846246 30895 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.857528 30842 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4854c286195847a89cac7db7426b1b6e" candidate_term: 5 candidate_status { last_received { term: 4 index: 7 } } ignore_live_leader: false dest_uuid: "58b878598d2e431ba4a31b17f243118d"
03:07:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.858105 30842 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 4 FOLLOWER]: Advancing to term 5
03:07:43.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:43.860066 30722 leader_election.cc:336] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093): Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111)
03:07:43.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.855901 30895 leader_election.cc:290] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 5 election: Requested vote from peers 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093), 58b878598d2e431ba4a31b17f243118d (127.28.188.252:43955)
03:07:43.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.867645 30842 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4854c286195847a89cac7db7426b1b6e in term 5.
03:07:43.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.868810 30722 leader_election.cc:304] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4854c286195847a89cac7db7426b1b6e, 58b878598d2e431ba4a31b17f243118d; no voters: 9362c8297aa946dfbcd513a95fa97f51
03:07:43.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.869843 30895 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 5 FOLLOWER]: Leader election won for term 5
03:07:43.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.877154 30895 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [term 5 LEADER]: Becoming Leader. State: Replica: 4854c286195847a89cac7db7426b1b6e, State: Running, Role: LEADER
03:07:43.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.878265 30895 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 4.7, Last appended by leader: 7, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } }
03:07:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.893996 30901 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4854c286195847a89cac7db7426b1b6e. Latest consensus state: current_term: 5 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:43.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.895848 30901 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: This master's current role is: LEADER
03:07:43.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.912248 30903 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:07:43.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.942024 30903 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:07:43.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.950829 30903 catalog_manager.cc:1269] Loaded cluster ID: 7b1010dc34f4435aa7dbf3461c94b984
03:07:43.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.954584 30903 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:07:43.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.965113 30903 catalog_manager.cc:1514] Loading token signing keys...
03:07:43.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.975219 30903 catalog_manager.cc:6055] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e: Loaded TSK: 0
03:07:43.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:43.978335 30903 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:07:44.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:44.026458 30864 messenger.cc:194] Invalid argument: socket address is ambiguous: no data on pending connections for acceptor pool at 127.28.188.252:43955
03:07:44.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:44.309804 30722 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e -> Peer 9362c8297aa946dfbcd513a95fa97f51 (127.28.188.253:44093): Couldn't send request to peer 9362c8297aa946dfbcd513a95fa97f51. Status: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:07:44.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:44.381943 30842 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [term 5 FOLLOWER]: Refusing update from remote peer 4854c286195847a89cac7db7426b1b6e: Log matching property violated. Preceding OpId in replica: term: 4 index: 7. Preceding OpId from leader: term: 5 index: 8. (index mismatch)
03:07:44.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:44.384631 30901 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [LEADER]: Connected to new peer: Peer: permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
03:07:44.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:44.432658 30905 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4854c286195847a89cac7db7426b1b6e. Latest consensus state: current_term: 5 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:44.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:44.433588 30905 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: This master's current role is: FOLLOWER
03:07:44.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:44.452513 30905 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 5 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:44.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:44.453269 30905 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58b878598d2e431ba4a31b17f243118d [sys.catalog]: This master's current role is: FOLLOWER
03:07:44.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:44.455848 30901 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 5 leader_uuid: "4854c286195847a89cac7db7426b1b6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4854c286195847a89cac7db7426b1b6e" member_type: VOTER last_known_addr { host: "127.28.188.254" port: 39373 } } peers { permanent_uuid: "9362c8297aa946dfbcd513a95fa97f51" member_type: VOTER last_known_addr { host: "127.28.188.253" port: 44093 } } peers { permanent_uuid: "58b878598d2e431ba4a31b17f243118d" member_type: VOTER last_known_addr { host: "127.28.188.252" port: 43955 } } }
03:07:44.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:44.456645 30901 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4854c286195847a89cac7db7426b1b6e [sys.catalog]: This master's current role is: LEADER
03:07:45.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:45.138227 30891 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 30869
03:07:45.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:45.749800 30869 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.013s user 0.490s sys 0.861s
03:07:45.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:43.738219 30893 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:45.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:45.750159 30869 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.013s user 0.490s sys 0.861s
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:45.759977 30918 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.759924 30869 server_base.cc:1061] running on GCE node
03:07:45.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.761790 30869 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.765661 30869 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:45.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.767295 30869 hybrid_clock.cc:648] HybridClock initialized: now 1773976065767158 us; error 134 us; skew 500 ppm
03:07:45.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.768244 30869 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:07:45 dist-test-slave-sx75 krb5kdc[29438](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773976065, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.188.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:45.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.776047 30869 init.cc:377] Logged in from keytab as kudu/127.28.188.253@KRBTEST.COM (short username kudu)
03:07:45.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.781883 30869 webserver.cc:492] Webserver started at http://127.28.188.253:36353/ using document root <none> and password file <none>
03:07:45.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.783169 30869 fs_manager.cc:362] Metadata directory not provided
03:07:45.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.783463 30869 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:45.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.795136 30869 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.005s sys 0.001s
03:07:45.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.800980 30927 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:45.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.802371 30869 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
03:07:45.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.802816 30869 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:45.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9362c8297aa946dfbcd513a95fa97f51"
03:07:45.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:07:07 on dist-test-slave-sx75"
03:07:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.805612 30869 fs_report.cc:389] FS layout report
03:07:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/wal
03:07:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/data
03:07:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:45.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:45.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:45.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:45.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.866348 30697 master_runner.cc:186] Error getting master registration for 127.28.188.253:44093: Network error: Client connection negotiation failed: client connection to 127.28.188.253:44093: connect: Connection refused (error 111),
03:07:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.867009 30697 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.879563 30869 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.890879 30869 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.891506 30869 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:45.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.984078 30869 rpc_server.cc:307] RPC server started. Bound to: 127.28.188.253:44093
03:07:45.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.984190 30978 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.188.253:44093 every 8 connection(s)
03:07:45.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.988224 30869 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/master-1/data/info.pb
03:07:45.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:45.990307 29427 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 30869
03:07:45.991 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:45.991 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.188.254:39373(127.28.188.254:39373)] connecting to peer
03:07:46.005 [DEBUG - kudu-nio-5] (Connection.java:757) [peer master-127.28.188.254:39373(127.28.188.254:39373)] Successfully connected to peer
03:07:46.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:46.008067 30979 sys_catalog.cc:263] Verifying existing consensus state
03:07:46.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:46.019412 30979 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Bootstrap starting.
03:07:46.039 [DEBUG - kudu-nio-5] (SecurityContext.java:222) Refreshing Kerberos credentials...
03:07:46.039 [DEBUG - kudu-nio-5] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster4224814609473106497/krb5kdc/krb5cc
03:07:46.043 [DEBUG - kudu-nio-5] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:07:46.043 [WARN - kudu-nio-5] (SecurityContext.java:231) Tried to refresh Kerberos credentials but was unable to re-login from ticket cache
03:07:46.043 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.28.188.254:39373(127.28.188.254:39373)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
03:07:46.047 [DEBUG - main] (Connection.java:575) [peer master-127.28.188.252:43955(127.28.188.252:43955)] disconnecting while in state TERMINATED
03:07:46.047 [DEBUG - main] (Connection.java:575) [peer master-127.28.188.254:39373(127.28.188.254:39373)] disconnecting while in state TERMINATED
03:07:46.047 [DEBUG - main] (Connection.java:575) [peer master-127.28.188.253:44093(127.28.188.253:44093)] disconnecting while in state TERMINATED
03:07:46.048 [DEBUG - kudu-nio-5] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:07:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:46.050524 30982 negotiation.cc:337] Failed RPC negotiation. Trace:
03:07:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:07:45.997574 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:54028 (local address 127.28.188.254:39373)
03:07:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:07:46.029327 (+ 31753us) server_negotiation.cc:207] Beginning negotiation
03:07:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:07:46.029373 (+ 46us) server_negotiation.cc:400] Waiting for connection header
03:07:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:07:46.034769 (+ 5396us) server_negotiation.cc:408] Connection header received
03:07:46.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:07:46.035090 (+ 321us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
03:07:46.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:07:46.035122 (+ 32us) server_negotiation.cc:462] Received NEGOTIATE request from client
03:07:46.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:07:46.035441 (+ 319us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
03:07:46.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:07:46.036750 (+ 1309us) server_negotiation.cc:227] Negotiated authn=SASL
03:07:46.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:07:46.049075 (+ 12325us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:54028: BlockingRecv error: recv got EOF from 127.0.0.1:54028 (error 108)
03:07:46.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":31042,"thread_start_us":31826,"threads_started":1}
03:07:46.055 [DEBUG - kudu-nio-5] (Connection.java:239) [peer master-127.28.188.254:39373(127.28.188.254:39373)] handling channelInactive
03:07:46.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:46.055624 29427 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 30697
03:07:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:46.078687 30979 log.cc:826] T 00000000000000000000000000000000 P 9362c8297aa946dfbcd513a95fa97f51: Log is configured to *not* fsync() on all Append() calls
03:07:46.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:46.091562 29427 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 30869
03:07:46.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:46.113356 29427 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 30777
03:07:46.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:46.596518 29412 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8080 after lost signal to thread 29285
03:07:46.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:07:46.597393 29412 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8c00 after lost signal to thread 29415
03:07:47.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-20T03:07:47Z chronyd exiting
03:07:48.379 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:07:48.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:48.383715 28691 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 28994
03:07:48.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:48.415165 28691 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 29136
03:07:48.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:48.446918 28691 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 29284
03:07:48.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:48.475703 28691 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 28703
03:07:48.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:48.511853 28691 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 28775
03:07:48.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:07:48.547586 28691 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 28851
03:07:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-20T03:07:49Z chronyd exiting
03:07:50.777 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more