16:29:19.829 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
16:29:19.829 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskMUvRhB/test-tmp
16:29:19.829 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
16:29:19.830 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskMUvRhB/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
16:29:19.834 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
16:29:20.332 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:29:20.332 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:29:20.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:20.337392 1836 mini_oidc.cc:150] Starting JWKS server
16:29:20.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:20.342700 1836 webserver.cc:492] Webserver started at http://127.0.0.1:33707/ using document root <none> and password file <none>
16:29:20.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:20.343305 1836 mini_oidc.cc:186] Starting OIDC Discovery server
16:29:20.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:20.345263 1836 webserver.cc:492] Webserver started at http://127.0.0.1:36075/ using document root <none> and password file <none>
16:29:20.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-17T16:29:20Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
16:29:20.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-17T16:29:20Z Disabled control of system clock
16:29:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:20.429159 1836 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-0/wal
16:29:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-0/data
16:29:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:29:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:29:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:29:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-0/logs
16:29:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-0/data/info.pb
16:29:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.203.62:41027
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.203.62
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.203.20:33431
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.203.62:41027,127.1.203.61:40551,127.1.203.60:46495
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:29:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33707/jwks.json with env {}
16:29:20.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:20.918715 1848 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:29:20.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:20.919472 1848 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:29:20.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:20.920514 1848 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:29:20.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:20.968336 1848 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:29:20.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:20.968667 1848 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:29:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:20.970345 1848 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:29:20.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:20.970806 1848 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:29:20.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:20.971047 1848 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:29:20.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:20.971228 1848 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33707/jwks.json
16:29:21.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:21.017946 1848 master_runner.cc:386] Master server non-default flags:
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.203.20:33431
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-0/data
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-0/wal
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.203.62:41027,127.1.203.61:40551,127.1.203.60:46495
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.203.62:41027
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33707/jwks.json
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-0/data/info.pb
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.203.62
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-0/logs
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:29:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:21.019484 1848 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:29:21.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:21.021347 1848 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:29:21.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:21.037835 1854 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:22.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:21.039456 1855 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:22.426584 1858 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:22.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:22.431660 1857 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1385 milliseconds
16:29:22.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.431815 1848 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:29:22.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.433216 1848 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:29:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.436724 1848 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:29:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.438342 1848 hybrid_clock.cc:648] HybridClock initialized: now 1773764962438281 us; error 75 us; skew 500 ppm
16:29:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.439421 1848 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:29:22.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.452497 1848 webserver.cc:492] Webserver started at http://127.1.203.62:40371/ using document root <none> and password file <none>
16:29:22.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.453464 1848 fs_manager.cc:362] Metadata directory not provided
16:29:22.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.453666 1848 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:29:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.454123 1848 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:29:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.458652 1848 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-0/data/instance:
16:29:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c6539747b8fe4235b67200c0648fe78e"
16:29:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:22 on dist-test-slave-jdhl"
16:29:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.459906 1848 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-0/wal/instance:
16:29:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c6539747b8fe4235b67200c0648fe78e"
16:29:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:22 on dist-test-slave-jdhl"
16:29:22.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.468429 1848 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.003s sys 0.004s
16:29:22.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.475135 1865 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:29:22.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.476728 1848 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
16:29:22.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.477136 1848 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-0/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-0/wal
16:29:22.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c6539747b8fe4235b67200c0648fe78e"
16:29:22.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:22 on dist-test-slave-jdhl"
16:29:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.477555 1848 fs_report.cc:389] FS layout report
16:29:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:29:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-0/wal
16:29:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-0/wal
16:29:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-0/data/data
16:29:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:29:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:29:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:29:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:29:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:29:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:29:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:29:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:29:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:29:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:29:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:29:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.554970 1848 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:29:22.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.559788 1848 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:29:22.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.561048 1848 kserver.cc:163] Server-wide thread pool size limit: 3276
16:29:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.676838 1848 rpc_server.cc:307] RPC server started. Bound to: 127.1.203.62:41027
16:29:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.676959 1916 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.203.62:41027 every 8 connection(s)
16:29:22.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.680305 1848 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-0/data/info.pb
16:29:22.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.685873 1836 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 1848
16:29:22.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.687681 1836 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-0/wal/instance
16:29:22.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.691115 1917 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.693632 1836 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-1/wal
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-1/data
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-1/logs
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-1/data/info.pb
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.203.61:40551
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.203.61
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.203.20:33431
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:22.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:22.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:29:22.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.203.62:41027,127.1.203.61:40551,127.1.203.60:46495
16:29:22.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:29:22.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33707/jwks.json with env {}
16:29:22.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.705415 1917 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } has no permanent_uuid. Determining permanent_uuid...
16:29:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.733963 1917 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } has no permanent_uuid. Determining permanent_uuid...
16:29:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:22.736599 1868 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.203.61:40551: connect: Connection refused (error 111)
16:29:22.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:22.741317 1917 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.203.61:40551: Network error: Client connection negotiation failed: client connection to 127.1.203.61:40551: connect: Connection refused (error 111)
16:29:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.790889 1917 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } attempt: 1
16:29:22.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:22.803368 1917 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.203.61:40551: Network error: Client connection negotiation failed: client connection to 127.1.203.61:40551: connect: Connection refused (error 111)
16:29:22.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.872126 1917 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } attempt: 2
16:29:22.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:22.876715 1917 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.203.61:40551: Network error: Client connection negotiation failed: client connection to 127.1.203.61:40551: connect: Connection refused (error 111)
16:29:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:22.968437 1917 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } attempt: 3
16:29:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:22.973230 1917 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.203.61:40551: Network error: Client connection negotiation failed: client connection to 127.1.203.61:40551: connect: Connection refused (error 111)
16:29:23.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:23.071049 1919 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:29:23.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:23.071817 1919 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:29:23.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:23.072424 1919 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:29:23.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:23.108211 1919 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:29:23.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:23.108608 1919 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:29:23.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:23.108958 1919 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:29:23.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:23.109277 1919 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:29:23.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:23.109580 1919 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:29:23.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:23.109825 1919 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33707/jwks.json
16:29:23.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:23.116665 1917 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } attempt: 4
16:29:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:23.120944 1917 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.203.61:40551: Network error: Client connection negotiation failed: client connection to 127.1.203.61:40551: connect: Connection refused (error 111)
16:29:23.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:23.150739 1919 master_runner.cc:386] Master server non-default flags:
16:29:23.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:23.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.203.20:33431
16:29:23.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-1/data
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-1/wal
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.203.62:41027,127.1.203.61:40551,127.1.203.60:46495
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.203.61:40551
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33707/jwks.json
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-1/data/info.pb
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.203.61
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:29:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:29:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:29:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-1/logs
16:29:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:29:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:29:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:29:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:29:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:29:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:29:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:29:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:29:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:29:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:23.152853 1919 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:29:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:23.155311 1919 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:29:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:23.170553 1928 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:23.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:23.420452 1917 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } attempt: 5
16:29:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:23.430295 1917 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.203.61:40551: Network error: Client connection negotiation failed: client connection to 127.1.203.61:40551: connect: Connection refused (error 111)
16:29:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:23.978075 1917 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } attempt: 6
16:29:24.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:24.016122 1917 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.203.61:40551: Network error: Client connection negotiation failed: client connection to 127.1.203.61:40551: connect: Connection refused (error 111)
16:29:25.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.076874 1917 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } attempt: 7
16:29:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.109714 1917 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.203.61:40551: Network error: Client connection negotiation failed: client connection to 127.1.203.61:40551: connect: Connection refused (error 111)
16:29:25.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:24.574667 1927 debug-util.cc:398] Leaking SignalData structure 0x7b080002edc0 after lost signal to thread 1919
16:29:25.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.116868 1919 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.947s user 0.687s sys 1.246s
16:29:25.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.117311 1919 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.948s user 0.687s sys 1.246s
16:29:25.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:23.171237 1930 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.120338 1932 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:25.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.124826 1931 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1948 milliseconds
16:29:25.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.124884 1919 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:29:25.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.126312 1919 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:29:25.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.129494 1919 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:29:25.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.130954 1919 hybrid_clock.cc:648] HybridClock initialized: now 1773764965130891 us; error 59 us; skew 500 ppm
16:29:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.131938 1919 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:29:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.138924 1919 webserver.cc:492] Webserver started at http://127.1.203.61:45239/ using document root <none> and password file <none>
16:29:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.140053 1919 fs_manager.cc:362] Metadata directory not provided
16:29:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.140349 1919 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:29:25.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.140883 1919 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:29:25.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.146008 1919 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-1/data/instance:
16:29:25.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f"
16:29:25.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:25 on dist-test-slave-jdhl"
16:29:25.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.147727 1919 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-1/wal/instance:
16:29:25.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f"
16:29:25.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:25 on dist-test-slave-jdhl"
16:29:25.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.156770 1919 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.008s sys 0.000s
16:29:25.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.164022 1943 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:29:25.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.167322 1919 fs_manager.cc:730] Time spent opening block manager: real 0.007s user 0.004s sys 0.003s
16:29:25.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.167809 1919 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-1/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-1/wal
16:29:25.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f"
16:29:25.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:25 on dist-test-slave-jdhl"
16:29:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.168427 1919 fs_report.cc:389] FS layout report
16:29:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:29:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-1/wal
16:29:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-1/wal
16:29:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-1/data/data
16:29:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:29:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:29:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:29:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:29:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:29:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:29:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:29:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:29:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:29:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:29:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:29:25.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.238106 1919 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:29:25.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.239799 1919 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:29:25.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.240391 1919 kserver.cc:163] Server-wide thread pool size limit: 3276
16:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.321043 1919 rpc_server.cc:307] RPC server started. Bound to: 127.1.203.61:40551
16:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.321125 1994 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.203.61:40551 every 8 connection(s)
16:29:25.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.324167 1919 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-1/data/info.pb
16:29:25.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.327879 1836 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 1919
16:29:25.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.328647 1836 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-1/wal/instance
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.332350 1836 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-2/wal
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-2/data
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-2/logs
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-2/data/info.pb
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.203.60:46495
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.203.60
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.203.20:33431
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.203.62:41027,127.1.203.61:40551,127.1.203.60:46495
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33707/jwks.json with env {}
16:29:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.333702 1995 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:29:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.348536 1995 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } has no permanent_uuid. Determining permanent_uuid...
16:29:25.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.378587 1995 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } has no permanent_uuid. Determining permanent_uuid...
16:29:25.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.395129 1995 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } has no permanent_uuid. Determining permanent_uuid...
16:29:25.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.397657 1946 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.203.60:46495: connect: Connection refused (error 111)
16:29:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.402604 1995 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.203.60:46495: Network error: Client connection negotiation failed: client connection to 127.1.203.60:46495: connect: Connection refused (error 111)
16:29:25.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.452282 1995 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } attempt: 1
16:29:25.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.457901 1995 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.203.60:46495: Network error: Client connection negotiation failed: client connection to 127.1.203.60:46495: connect: Connection refused (error 111)
16:29:25.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.526557 1995 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } attempt: 2
16:29:25.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.531272 1995 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.203.60:46495: Network error: Client connection negotiation failed: client connection to 127.1.203.60:46495: connect: Connection refused (error 111)
16:29:25.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.622879 1995 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } attempt: 3
16:29:25.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.627473 1995 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.203.60:46495: Network error: Client connection negotiation failed: client connection to 127.1.203.60:46495: connect: Connection refused (error 111)
16:29:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.679510 1997 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:29:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.680202 1997 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:29:25.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.680812 1997 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:29:25.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.715562 1997 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:29:25.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.715955 1997 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:29:25.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.716320 1997 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:29:25.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.716629 1997 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:29:25.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.716953 1997 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:29:25.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.717204 1997 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33707/jwks.json
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.755704 1997 master_runner.cc:386] Master server non-default flags:
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.203.20:33431
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-2/data
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-2/wal
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.203.62:41027,127.1.203.61:40551,127.1.203.60:46495
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.203.60:46495
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33707/jwks.json
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-2/data/info.pb
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.203.60
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:29:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:29:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:29:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:29:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-2/logs
16:29:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:29:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:29:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:29:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:29:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:29:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:29:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:29:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:29:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:29:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.757470 1997 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:29:25.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.759254 1997 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:29:25.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.772388 1995 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } attempt: 4
16:29:25.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.776789 2008 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:25.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.780228 1995 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.203.60:46495: Network error: Client connection negotiation failed: client connection to 127.1.203.60:46495: connect: Connection refused (error 111)
16:29:26.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:26.079732 1995 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } attempt: 5
16:29:26.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:26.084604 1995 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.203.60:46495: Network error: Client connection negotiation failed: client connection to 127.1.203.60:46495: connect: Connection refused (error 111)
16:29:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:26.632393 1995 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } attempt: 6
16:29:26.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:26.641108 1995 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.203.60:46495: Network error: Client connection negotiation failed: client connection to 127.1.203.60:46495: connect: Connection refused (error 111)
16:29:27.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.777937 2009 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:27.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:25.780850 2011 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:25.781670 1997 server_base.cc:1060] running on GCE node
16:29:27.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.096760 1997 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:29:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.100474 1997 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:29:27.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.102298 1997 hybrid_clock.cc:648] HybridClock initialized: now 1773764967101924 us; error 185 us; skew 500 ppm
16:29:27.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.103641 1997 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:29:27.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.111044 1997 webserver.cc:492] Webserver started at http://127.1.203.60:44643/ using document root <none> and password file <none>
16:29:27.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.112313 1997 fs_manager.cc:362] Metadata directory not provided
16:29:27.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.112668 1997 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:29:27.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.113464 1997 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:29:27.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.118461 1997 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-2/data/instance:
16:29:27.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "960b40c8905e461bad3454144ce157f4"
16:29:27.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:27 on dist-test-slave-jdhl"
16:29:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.119885 1997 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-2/wal/instance:
16:29:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "960b40c8905e461bad3454144ce157f4"
16:29:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:27 on dist-test-slave-jdhl"
16:29:27.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.129944 1997 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.005s sys 0.004s
16:29:27.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.137629 2021 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:29:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.139320 1997 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.003s
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.139704 1997 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-2/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-2/wal
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "960b40c8905e461bad3454144ce157f4"
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:27 on dist-test-slave-jdhl"
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.140061 1997 fs_report.cc:389] FS layout report
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-2/wal
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-2/wal
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-2/data/data
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:29:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.200502 1917 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } attempt: 8
16:29:27.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.210665 1997 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:29:27.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.212797 1997 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:29:27.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.213512 1997 kserver.cc:163] Server-wide thread pool size limit: 3276
16:29:27.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.214953 1917 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } has no permanent_uuid. Determining permanent_uuid...
16:29:27.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:27.221674 1917 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.203.60:46495: Network error: Client connection negotiation failed: client connection to 127.1.203.60:46495: connect: Connection refused (error 111)
16:29:27.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.287310 1917 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } attempt: 1
16:29:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:27.293591 1917 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.203.60:46495: Network error: Client connection negotiation failed: client connection to 127.1.203.60:46495: connect: Connection refused (error 111)
16:29:27.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.301270 1997 rpc_server.cc:307] RPC server started. Bound to: 127.1.203.60:46495
16:29:27.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.301338 2075 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.203.60:46495 every 8 connection(s)
16:29:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.305061 1997 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-2/data/info.pb
16:29:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.309677 1836 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 1997
16:29:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.310215 1836 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/master-2/wal/instance
16:29:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.313001 2076 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:29:27.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.330636 2076 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } has no permanent_uuid. Determining permanent_uuid...
16:29:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.347170 1917 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } attempt: 2
16:29:27.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.375073 2076 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } has no permanent_uuid. Determining permanent_uuid...
16:29:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.389679 2076 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } has no permanent_uuid. Determining permanent_uuid...
16:29:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.390017 1917 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e: Bootstrap starting.
16:29:27.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.397063 1917 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e: Neither blocks nor log segments found. Creating new log.
16:29:27.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.400024 1917 log.cc:826] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e: Log is configured to *not* fsync() on all Append() calls
16:29:27.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.406430 1917 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e: No bootstrap required, opened a new log
16:29:27.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.414148 2076 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4: Bootstrap starting.
16:29:27.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.424836 2076 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4: Neither blocks nor log segments found. Creating new log.
16:29:27.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.428344 2076 log.cc:826] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4: Log is configured to *not* fsync() on all Append() calls
16:29:27.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.436050 2076 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4: No bootstrap required, opened a new log
16:29:27.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.440237 1917 raft_consensus.cc:359] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } }
16:29:27.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.443218 1917 raft_consensus.cc:385] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:29:27.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.443565 1917 raft_consensus.cc:740] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6539747b8fe4235b67200c0648fe78e, State: Initialized, Role: FOLLOWER
16:29:27.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.445160 1917 consensus_queue.cc:260] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } }
16:29:27.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.448796 2086 sys_catalog.cc:455] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } } }
16:29:27.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.449610 2086 sys_catalog.cc:458] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e [sys.catalog]: This master's current role is: FOLLOWER
16:29:27.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.451627 1917 sys_catalog.cc:565] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e [sys.catalog]: configured and running, proceeding with master startup.
16:29:27.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.471611 2076 raft_consensus.cc:359] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } }
16:29:27.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.472905 2076 raft_consensus.cc:385] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:29:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.473492 2076 raft_consensus.cc:740] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 960b40c8905e461bad3454144ce157f4, State: Initialized, Role: FOLLOWER
16:29:27.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.474746 2076 consensus_queue.cc:260] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } }
16:29:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.478338 2097 sys_catalog.cc:455] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } } }
16:29:27.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.479776 2097 sys_catalog.cc:458] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [sys.catalog]: This master's current role is: FOLLOWER
16:29:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.481453 2076 sys_catalog.cc:565] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [sys.catalog]: configured and running, proceeding with master startup.
16:29:27.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:27.492659 2098 catalog_manager.cc:1568] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:29:27.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:27.493111 2098 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:29:27.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.490870 1848 master_runner.cc:186] Error getting master registration for 127.1.203.61:40551: OK, instance_id { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" instance_seqno: 1773764965298654 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:29:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.526005 1848 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:29:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.531538 2097 raft_consensus.cc:493] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:29:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.532181 2097 raft_consensus.cc:515] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } }
16:29:27.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.536218 2097 leader_election.cc:290] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c6539747b8fe4235b67200c0648fe78e (127.1.203.62:41027), 119a8ed141a24b7bb7ebb90d9d51fd0f (127.1.203.61:40551)
16:29:27.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.538198 1892 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "960b40c8905e461bad3454144ce157f4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6539747b8fe4235b67200c0648fe78e" is_pre_election: true
16:29:27.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.539208 1892 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 960b40c8905e461bad3454144ce157f4 in term 0.
16:29:27.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.541102 2023 leader_election.cc:304] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 960b40c8905e461bad3454144ce157f4, c6539747b8fe4235b67200c0648fe78e; no voters:
16:29:27.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.542054 2097 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [term 0 FOLLOWER]: Leader pre-election won for term 1
16:29:27.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.542403 2097 raft_consensus.cc:493] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:29:27.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.542735 2097 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [term 0 FOLLOWER]: Advancing to term 1
16:29:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.552596 2097 raft_consensus.cc:515] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } }
16:29:27.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.555893 1892 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "960b40c8905e461bad3454144ce157f4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6539747b8fe4235b67200c0648fe78e"
16:29:27.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.556548 1892 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e [term 0 FOLLOWER]: Advancing to term 1
16:29:27.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:27.559089 2110 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:29:27.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:27.559460 2110 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:29:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.568370 1892 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 960b40c8905e461bad3454144ce157f4 in term 1.
16:29:27.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.569763 2097 leader_election.cc:290] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [CANDIDATE]: Term 1 election: Requested vote from peers c6539747b8fe4235b67200c0648fe78e (127.1.203.62:41027), 119a8ed141a24b7bb7ebb90d9d51fd0f (127.1.203.61:40551)
16:29:27.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.576900 2023 leader_election.cc:304] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 960b40c8905e461bad3454144ce157f4, c6539747b8fe4235b67200c0648fe78e; no voters:
16:29:27.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:27.580500 1871 tablet.cc:2379] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:29:27.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.581334 2097 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [term 1 FOLLOWER]: Leader election won for term 1
16:29:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.590444 2097 raft_consensus.cc:697] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [term 1 LEADER]: Becoming Leader. State: Replica: 960b40c8905e461bad3454144ce157f4, State: Running, Role: LEADER
16:29:27.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.592010 2097 consensus_queue.cc:237] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } }
16:29:27.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.540783 1997 master_runner.cc:186] Error getting master registration for 127.1.203.61:40551: OK, instance_id { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" instance_seqno: 1773764965298654 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:29:27.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.605463 1997 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:29:27.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.619702 2112 sys_catalog.cc:455] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 960b40c8905e461bad3454144ce157f4. Latest consensus state: current_term: 1 leader_uuid: "960b40c8905e461bad3454144ce157f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } } }
16:29:27.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.620605 2112 sys_catalog.cc:458] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [sys.catalog]: This master's current role is: LEADER
16:29:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.625623 2114 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
16:29:27.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.634407 2114 catalog_manager.cc:1494] Initializing Kudu cluster ID...
16:29:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.684989 1892 raft_consensus.cc:1275] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e [term 1 FOLLOWER]: Refusing update from remote peer 960b40c8905e461bad3454144ce157f4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:29:27.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.689755 2112 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
16:29:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.702373 1995 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } attempt: 7
16:29:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:27.721000 2030 tablet.cc:2379] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:29:27.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.738721 2086 sys_catalog.cc:455] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 960b40c8905e461bad3454144ce157f4. Latest consensus state: current_term: 1 leader_uuid: "960b40c8905e461bad3454144ce157f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } } }
16:29:27.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.740031 2086 sys_catalog.cc:458] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e [sys.catalog]: This master's current role is: FOLLOWER
16:29:27.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.750461 2115 mvcc.cc:204] Tried to move back new op lower bound from 7265341307599650816 to 7265341307268272128. Current Snapshot: MvccSnapshot[applied={T|T < 7265341307599650816}]
16:29:27.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.761638 2086 sys_catalog.cc:455] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "960b40c8905e461bad3454144ce157f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } } }
16:29:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.764111 2086 sys_catalog.cc:458] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e [sys.catalog]: This master's current role is: FOLLOWER
16:29:27.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.769503 2112 sys_catalog.cc:455] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "960b40c8905e461bad3454144ce157f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } } }
16:29:27.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.770423 2112 sys_catalog.cc:458] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [sys.catalog]: This master's current role is: LEADER
16:29:27.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.797798 1995 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f: Bootstrap starting.
16:29:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.799440 2114 catalog_manager.cc:1357] Generated new cluster ID: 644a457aa83d42ae8ed257f14e97156f
16:29:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.799794 2114 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
16:29:27.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.813906 1995 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f: Neither blocks nor log segments found. Creating new log.
16:29:27.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.817816 1995 log.cc:826] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f: Log is configured to *not* fsync() on all Append() calls
16:29:27.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.839797 1995 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f: No bootstrap required, opened a new log
16:29:27.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.875792 2114 catalog_manager.cc:1380] Generated new certificate authority record
16:29:27.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.893450 2114 catalog_manager.cc:1514] Loading token signing keys...
16:29:27.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.928382 1995 raft_consensus.cc:359] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } }
16:29:27.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.929330 1995 raft_consensus.cc:385] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:29:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.929675 1995 raft_consensus.cc:740] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 119a8ed141a24b7bb7ebb90d9d51fd0f, State: Initialized, Role: FOLLOWER
16:29:27.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.931000 1995 consensus_queue.cc:260] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } }
16:29:27.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.936695 1995 sys_catalog.cc:565] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f [sys.catalog]: configured and running, proceeding with master startup.
16:29:27.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.939677 1968 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f [term 0 FOLLOWER]: Advancing to term 1
16:29:27.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.944562 1969 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "960b40c8905e461bad3454144ce157f4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f"
16:29:27.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.946475 1970 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "960b40c8905e461bad3454144ce157f4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" is_pre_election: true
16:29:27.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.948226 2114 catalog_manager.cc:6044] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4: Generated new TSK 0
16:29:27.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.949477 2114 catalog_manager.cc:1524] Initializing in-progress tserver states...
16:29:27.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.956594 1968 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f [term 1 FOLLOWER]: Refusing update from remote peer 960b40c8905e461bad3454144ce157f4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:29:27.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.958557 2121 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
16:29:27.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.935541 2123 sys_catalog.cc:455] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } } }
16:29:27.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.960330 2123 sys_catalog.cc:458] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f [sys.catalog]: This master's current role is: FOLLOWER
16:29:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:27.986654 1836 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-0/wal
16:29:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-0/data
16:29:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:29:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:29:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:29:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-0/logs
16:29:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-0/data/info.pb
16:29:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:29:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:29:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:29:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:29:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:29:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:29:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.203.1:0
16:29:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.203.1
16:29:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.203.1
16:29:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:29:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.203.62:41027,127.1.203.61:40551,127.1.203.60:46495
16:29:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.203.20:33431
16:29:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:29:28.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:28.016678 2123 sys_catalog.cc:455] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 960b40c8905e461bad3454144ce157f4. Latest consensus state: current_term: 1 leader_uuid: "960b40c8905e461bad3454144ce157f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } } }
16:29:28.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:28.016908 2129 sys_catalog.cc:455] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "960b40c8905e461bad3454144ce157f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } } }
16:29:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:28.017915 2123 sys_catalog.cc:458] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f [sys.catalog]: This master's current role is: FOLLOWER
16:29:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:28.018052 2129 sys_catalog.cc:458] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f [sys.catalog]: This master's current role is: FOLLOWER
16:29:28.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:28.132910 2140 catalog_manager.cc:1269] Loaded cluster ID: 644a457aa83d42ae8ed257f14e97156f
16:29:28.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:28.133250 2140 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f: loading cluster ID for follower catalog manager: success
16:29:28.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:28.152963 2140 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f: acquiring CA information for follower catalog manager: success
16:29:28.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:28.158836 2140 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 119a8ed141a24b7bb7ebb90d9d51fd0f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:29:28.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:28.381201 2121 sys_catalog.cc:455] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "960b40c8905e461bad3454144ce157f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6539747b8fe4235b67200c0648fe78e" member_type: VOTER last_known_addr { host: "127.1.203.62" port: 41027 } } peers { permanent_uuid: "119a8ed141a24b7bb7ebb90d9d51fd0f" member_type: VOTER last_known_addr { host: "127.1.203.61" port: 40551 } } peers { permanent_uuid: "960b40c8905e461bad3454144ce157f4" member_type: VOTER last_known_addr { host: "127.1.203.60" port: 46495 } } }
16:29:28.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:28.382009 2121 sys_catalog.cc:458] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 [sys.catalog]: This master's current role is: LEADER
16:29:28.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:28.500878 2098 catalog_manager.cc:1269] Loaded cluster ID: 644a457aa83d42ae8ed257f14e97156f
16:29:28.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:28.503259 2098 catalog_manager.cc:1562] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e: loading cluster ID for follower catalog manager: success
16:29:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:28.525738 2098 catalog_manager.cc:1584] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e: acquiring CA information for follower catalog manager: success
16:29:28.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:28.532606 2098 catalog_manager.cc:1612] T 00000000000000000000000000000000 P c6539747b8fe4235b67200c0648fe78e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:29:28.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:28.747128 2130 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:29:28.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:28.747824 2130 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:29:28.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:28.748728 2130 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:29:28.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:28.820358 2130 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:29:28.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:28.822002 2130 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.203.1
16:29:28.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:28.903198 2130 tablet_server_runner.cc:78] Tablet server non-default flags:
16:29:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.203.20:33431
16:29:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-0/data
16:29:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-0/wal
16:29:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.203.1:0
16:29:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:29:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-0/data/info.pb
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.203.1
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.203.62:41027,127.1.203.61:40551,127.1.203.60:46495
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.203.1
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-0/logs
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:29:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:28.905318 2130 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:29:28.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:28.907996 2130 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:29:28.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:28.944722 2148 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:30.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:28.958554 2152 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:30.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:28.959312 2130 server_base.cc:1060] running on GCE node
16:29:30.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:28.958326 2149 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:30.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.680397 2130 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:29:30.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.692056 2130 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:29:30.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.693526 2130 hybrid_clock.cc:648] HybridClock initialized: now 1773764970693484 us; error 55 us; skew 500 ppm
16:29:30.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.694697 2130 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:29:30.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.726747 2130 webserver.cc:492] Webserver started at http://127.1.203.1:36923/ using document root <none> and password file <none>
16:29:30.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.729089 2130 fs_manager.cc:362] Metadata directory not provided
16:29:30.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.730271 2130 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:29:30.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.731122 2130 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:29:30.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.747323 2130 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-0/data/instance:
16:29:30.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30426da51a024d4888bf27101d2b518e"
16:29:30.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:30 on dist-test-slave-jdhl"
16:29:30.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.749217 2130 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-0/wal/instance:
16:29:30.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30426da51a024d4888bf27101d2b518e"
16:29:30.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:30 on dist-test-slave-jdhl"
16:29:30.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.767935 2130 fs_manager.cc:696] Time spent creating directory manager: real 0.018s user 0.010s sys 0.008s
16:29:30.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.783775 2159 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:29:30.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.785300 2130 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.010s sys 0.001s
16:29:30.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.785759 2130 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-0/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-0/wal
16:29:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30426da51a024d4888bf27101d2b518e"
16:29:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:30 on dist-test-slave-jdhl"
16:29:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.786262 2130 fs_report.cc:389] FS layout report
16:29:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:29:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-0/wal
16:29:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-0/wal
16:29:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-0/data/data
16:29:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:29:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:29:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:29:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:29:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:29:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:29:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:29:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:29:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:29:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:29:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:29:30.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.952160 2130 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:29:30.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.954509 2130 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:29:30.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.955201 2130 kserver.cc:163] Server-wide thread pool size limit: 3276
16:29:30.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.968983 2130 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:29:30.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.982367 2130 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:29:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.982730 2130 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:29:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.983274 2130 ts_tablet_manager.cc:616] Registered 0 tablets
16:29:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:30.983508 2130 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:29:31.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.771556 2130 rpc_server.cc:307] RPC server started. Bound to: 127.1.203.1:45691
16:29:31.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.771680 2272 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.203.1:45691 every 8 connection(s)
16:29:31.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.774384 2130 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-0/data/info.pb
16:29:31.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.787129 1836 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 2130
16:29:31.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.788008 1836 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-0/wal/instance
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.804754 1836 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-1/wal
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-1/data
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-1/logs
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-1/data/info.pb
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.203.2:0
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.203.2
16:29:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.203.2
16:29:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:29:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.203.62:41027,127.1.203.61:40551,127.1.203.60:46495
16:29:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.203.20:33431
16:29:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:29:31.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.836851 2273 heartbeater.cc:344] Connected to a master server at 127.1.203.60:46495
16:29:31.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.837440 2273 heartbeater.cc:461] Registering TS with master...
16:29:31.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.839160 2273 heartbeater.cc:507] Master 127.1.203.60:46495 requested a full tablet report, sending...
16:29:31.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.843019 2274 heartbeater.cc:344] Connected to a master server at 127.1.203.61:40551
16:29:31.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.843412 2274 heartbeater.cc:461] Registering TS with master...
16:29:31.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.844396 2274 heartbeater.cc:507] Master 127.1.203.61:40551 requested a full tablet report, sending...
16:29:31.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.842554 2041 ts_manager.cc:194] Registered new tserver with Master: 30426da51a024d4888bf27101d2b518e (127.1.203.1:45691)
16:29:31.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.847800 1960 ts_manager.cc:194] Registered new tserver with Master: 30426da51a024d4888bf27101d2b518e (127.1.203.1:45691)
16:29:31.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.848531 2041 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.203.1:56723
16:29:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.857921 2279 heartbeater.cc:344] Connected to a master server at 127.1.203.62:41027
16:29:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.858431 2279 heartbeater.cc:461] Registering TS with master...
16:29:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.859351 2279 heartbeater.cc:507] Master 127.1.203.62:41027 requested a full tablet report, sending...
16:29:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:31.862520 1882 ts_manager.cc:194] Registered new tserver with Master: 30426da51a024d4888bf27101d2b518e (127.1.203.1:45691)
16:29:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:32.194265 2283 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:29:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:32.194857 2283 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:29:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:32.195353 2283 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:29:32.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:32.235944 2283 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:29:32.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:32.237166 2283 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.203.2
16:29:32.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:32.279424 2283 tablet_server_runner.cc:78] Tablet server non-default flags:
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.203.20:33431
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-1/data
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-1/wal
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.203.2:0
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-1/data/info.pb
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.203.2
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.203.62:41027,127.1.203.61:40551,127.1.203.60:46495
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.203.2
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-1/logs
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:29:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:29:32.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:32.280786 2283 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:29:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:32.282939 2283 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:29:32.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:32.298436 2290 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:32.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:32.855350 2273 heartbeater.cc:499] Master 127.1.203.60:46495 was elected leader, sending a full tablet report...
16:29:34.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:33.702896 2288 debug-util.cc:398] Leaking SignalData structure 0x7b080002eda0 after lost signal to thread 2283
16:29:34.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:34.216997 2288 kernel_stack_watchdog.cc:198] Thread 2283 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
16:29:34.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:29:34.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:29:34.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:29:34.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:29:34.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:32.299893 2291 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:34.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:34.218255 2283 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.918s user 0.711s sys 1.187s
16:29:34.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:34.219357 2283 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.919s user 0.711s sys 1.187s
16:29:34.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:34.220341 2292 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1917 milliseconds
16:29:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:34.222222 2293 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.222216 2283 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:29:34.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.230003 2283 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:29:34.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.233342 2283 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:29:34.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.234979 2283 hybrid_clock.cc:648] HybridClock initialized: now 1773764974234897 us; error 81 us; skew 500 ppm
16:29:34.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.235948 2283 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:29:34.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.243433 2283 webserver.cc:492] Webserver started at http://127.1.203.2:34803/ using document root <none> and password file <none>
16:29:34.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.244510 2283 fs_manager.cc:362] Metadata directory not provided
16:29:34.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.244796 2283 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:29:34.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.245342 2283 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:29:34.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.250631 2283 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-1/data/instance:
16:29:34.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c873e27bce114a2ba3aea3c5f688167a"
16:29:34.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:34 on dist-test-slave-jdhl"
16:29:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.252354 2283 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-1/wal/instance:
16:29:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c873e27bce114a2ba3aea3c5f688167a"
16:29:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:34 on dist-test-slave-jdhl"
16:29:34.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.260938 2283 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.004s sys 0.004s
16:29:34.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.270743 2301 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:29:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.272521 2283 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.004s sys 0.003s
16:29:34.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.273000 2283 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-1/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-1/wal
16:29:34.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c873e27bce114a2ba3aea3c5f688167a"
16:29:34.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:34 on dist-test-slave-jdhl"
16:29:34.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.273396 2283 fs_report.cc:389] FS layout report
16:29:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:29:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-1/wal
16:29:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-1/wal
16:29:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-1/data/data
16:29:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:29:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:29:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:29:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:29:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:29:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:29:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:29:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:29:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:29:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:29:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:29:34.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.343655 2283 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:29:34.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.345824 2283 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:29:34.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.346498 2283 kserver.cc:163] Server-wide thread pool size limit: 3276
16:29:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.351282 2283 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:29:34.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.357758 2283 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:29:34.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.358089 2283 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:29:34.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.358371 2283 ts_tablet_manager.cc:616] Registered 0 tablets
16:29:34.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.358577 2283 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:29:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.532912 2283 rpc_server.cc:307] RPC server started. Bound to: 127.1.203.2:38881
16:29:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.533072 2413 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.203.2:38881 every 8 connection(s)
16:29:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.535526 2283 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-1/data/info.pb
16:29:34.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.544539 1836 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 2283
16:29:34.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.545172 1836 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-1/wal/instance
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.574509 1836 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-2/wal
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-2/data
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-2/logs
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-2/data/info.pb
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.203.3:0
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.203.3
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.203.3
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.203.62:41027,127.1.203.61:40551,127.1.203.60:46495
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.203.20:33431
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:29:34.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.589946 2414 heartbeater.cc:344] Connected to a master server at 127.1.203.60:46495
16:29:34.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.590698 2414 heartbeater.cc:461] Registering TS with master...
16:29:34.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.592226 2414 heartbeater.cc:507] Master 127.1.203.60:46495 requested a full tablet report, sending...
16:29:34.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.595256 2041 ts_manager.cc:194] Registered new tserver with Master: c873e27bce114a2ba3aea3c5f688167a (127.1.203.2:38881)
16:29:34.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.596688 2041 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.203.2:36699
16:29:34.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.607081 2415 heartbeater.cc:344] Connected to a master server at 127.1.203.61:40551
16:29:34.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.607527 2415 heartbeater.cc:461] Registering TS with master...
16:29:34.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.608258 2415 heartbeater.cc:507] Master 127.1.203.61:40551 requested a full tablet report, sending...
16:29:34.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.608510 2416 heartbeater.cc:344] Connected to a master server at 127.1.203.62:41027
16:29:34.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.608836 2416 heartbeater.cc:461] Registering TS with master...
16:29:34.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.609423 2416 heartbeater.cc:507] Master 127.1.203.62:41027 requested a full tablet report, sending...
16:29:34.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.611186 1960 ts_manager.cc:194] Registered new tserver with Master: c873e27bce114a2ba3aea3c5f688167a (127.1.203.2:38881)
16:29:34.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:34.612462 1882 ts_manager.cc:194] Registered new tserver with Master: c873e27bce114a2ba3aea3c5f688167a (127.1.203.2:38881)
16:29:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:35.046928 2424 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:29:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:35.047538 2424 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:29:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:35.048141 2424 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:29:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:35.087637 2424 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:29:35.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:35.089578 2424 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.203.3
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:35.134914 2424 tablet_server_runner.cc:78] Tablet server non-default flags:
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.203.20:33431
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-2/data
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-2/wal
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.203.3:0
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-2/data/info.pb
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.203.3
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.203.62:41027,127.1.203.61:40551,127.1.203.60:46495
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.203.3
16:29:35.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-2/logs
16:29:35.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:35.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:29:35.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:29:35.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:29:35.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:29:35.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:29:35.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:29:35.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:29:35.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:29:35.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:29:35.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:35.136845 2424 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:29:35.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:35.139650 2424 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:29:35.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:35.157817 2431 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:35.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:35.600862 2414 heartbeater.cc:499] Master 127.1.203.60:46495 was elected leader, sending a full tablet report...
16:29:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:35.160079 2432 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:36.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:36.560248 2430 debug-util.cc:398] Leaking SignalData structure 0x7b080002eda0 after lost signal to thread 2424
16:29:36.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:36.700145 2433 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
16:29:36.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:36.699172 2424 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.541s user 0.589s sys 0.942s
16:29:36.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:36.701632 2434 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:36.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:36.701730 2424 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.544s user 0.590s sys 0.942s
16:29:36.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.702072 2424 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:29:36.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.706198 2424 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:29:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.708554 2424 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:29:36.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.710072 2424 hybrid_clock.cc:648] HybridClock initialized: now 1773764976710029 us; error 42 us; skew 500 ppm
16:29:36.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.711171 2424 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:29:36.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.718538 2424 webserver.cc:492] Webserver started at http://127.1.203.3:37615/ using document root <none> and password file <none>
16:29:36.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.720052 2424 fs_manager.cc:362] Metadata directory not provided
16:29:36.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.720368 2424 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:29:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.721001 2424 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:29:36.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.726112 2424 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-2/data/instance:
16:29:36.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b893ccd7ec7844fb9465c36cd1e656e3"
16:29:36.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:36 on dist-test-slave-jdhl"
16:29:36.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.727584 2424 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-2/wal/instance:
16:29:36.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b893ccd7ec7844fb9465c36cd1e656e3"
16:29:36.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:36 on dist-test-slave-jdhl"
16:29:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.735643 2424 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
16:29:36.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.742643 2442 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:29:36.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.744114 2424 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
16:29:36.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.744439 2424 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-2/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-2/wal
16:29:36.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b893ccd7ec7844fb9465c36cd1e656e3"
16:29:36.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:36 on dist-test-slave-jdhl"
16:29:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.744751 2424 fs_report.cc:389] FS layout report
16:29:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:29:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-2/wal
16:29:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-2/wal
16:29:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-2/data/data
16:29:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:29:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:29:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:29:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:29:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:29:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:29:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:29:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:29:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:29:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:29:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:29:36.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.799388 2424 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:29:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.801172 2424 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:29:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.801633 2424 kserver.cc:163] Server-wide thread pool size limit: 3276
16:29:36.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.805428 2424 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:29:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.810712 2424 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:29:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.811024 2424 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
16:29:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.811304 2424 ts_tablet_manager.cc:616] Registered 0 tablets
16:29:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:36.811452 2424 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:29:37.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.006754 2424 rpc_server.cc:307] RPC server started. Bound to: 127.1.203.3:45219
16:29:37.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.006932 2554 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.203.3:45219 every 8 connection(s)
16:29:37.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.010380 2424 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-2/data/info.pb
16:29:37.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.015298 1836 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 2424
16:29:37.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.015805 1836 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster858251856065061398/ts-2/wal/instance
16:29:37.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.064430 2556 heartbeater.cc:344] Connected to a master server at 127.1.203.61:40551
16:29:37.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.065050 2556 heartbeater.cc:461] Registering TS with master...
16:29:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.066468 2556 heartbeater.cc:507] Master 127.1.203.61:40551 requested a full tablet report, sending...
16:29:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.070053 1959 ts_manager.cc:194] Registered new tserver with Master: b893ccd7ec7844fb9465c36cd1e656e3 (127.1.203.3:45219)
16:29:37.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.072923 2555 heartbeater.cc:344] Connected to a master server at 127.1.203.60:46495
16:29:37.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.073313 2555 heartbeater.cc:461] Registering TS with master...
16:29:37.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.074276 2555 heartbeater.cc:507] Master 127.1.203.60:46495 requested a full tablet report, sending...
16:29:37.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.075510 2557 heartbeater.cc:344] Connected to a master server at 127.1.203.62:41027
16:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.075898 2557 heartbeater.cc:461] Registering TS with master...
16:29:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.076804 2557 heartbeater.cc:507] Master 127.1.203.62:41027 requested a full tablet report, sending...
16:29:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.076579 2041 ts_manager.cc:194] Registered new tserver with Master: b893ccd7ec7844fb9465c36cd1e656e3 (127.1.203.3:45219)
16:29:37.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.078353 2041 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.203.3:44001
16:29:37.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.079030 1882 ts_manager.cc:194] Registered new tserver with Master: b893ccd7ec7844fb9465c36cd1e656e3 (127.1.203.3:45219)
16:29:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.095621 1836 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
16:29:37.108 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:29:37.108 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
16:29:37.109 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.203.62"
port: 41027
}
bound_http_address {
host: "127.1.203.62"
port: 40371
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.203.61"
port: 40551
}
bound_http_address {
host: "127.1.203.61"
port: 45239
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.203.60"
port: 46495
}
bound_http_address {
host: "127.1.203.60"
port: 44643
}
}
}
16:29:37.109 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:29:37.111 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.1.203.1"
port: 45691
}
bound_http_address {
host: "127.1.203.1"
port: 36923
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.1.203.2"
port: 38881
}
bound_http_address {
host: "127.1.203.2"
port: 34803
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.1.203.3"
port: 45219
}
bound_http_address {
host: "127.1.203.3"
port: 37615
}
}
}
16:29:37.111 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
16:29:37.113 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster11824441528918273073/krb5kdc/krb5cc
16:29:37.113 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
16:29:37.114 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskMUvRhB/test-tmp
16:29:37.115 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
16:29:37.115 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskMUvRhB/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
16:29:37.117 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
16:29:37.522 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:29:37.522 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:29:37.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.526208 2566 mini_oidc.cc:150] Starting JWKS server
16:29:37.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.529764 2566 webserver.cc:492] Webserver started at http://127.0.0.1:33827/ using document root <none> and password file <none>
16:29:37.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.530359 2566 mini_oidc.cc:186] Starting OIDC Discovery server
16:29:37.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:37.532887 2566 webserver.cc:492] Webserver started at http://127.0.0.1:33485/ using document root <none> and password file <none>
16:29:37.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
16:29:37.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/principal' for realm 'KRBTEST.COM',
16:29:37.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
16:29:37.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:37 dist-test-slave-jdhl krb5kdc[2577](info): setting up network...
16:29:37.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
16:29:37.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:37 dist-test-slave-jdhl krb5kdc[2577](info): set up 2 sockets
16:29:37.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:37 dist-test-slave-jdhl krb5kdc[2577](info): commencing operation
16:29:37.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
16:29:38.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:38.082149 2555 heartbeater.cc:499] Master 127.1.203.60:46495 was elected leader, sending a full tablet report...
16:29:39.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:39.979745 2566 mini_kdc.cc:121] Time spent starting KDC: real 2.401s user 0.009s sys 0.017s
16:29:39.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
16:29:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:29:40.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
16:29:40.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
16:29:40.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:29:40.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
16:29:40.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
16:29:40.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:29:40.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
16:29:40.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:29:40.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/test-user.keytab.
16:29:40.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/test-user.keytab.
16:29:40.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:40 dist-test-slave-jdhl krb5kdc[2577](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764980, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:29:40.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:29:40.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-17T16:29:40Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
16:29:40.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-17T16:29:40Z Disabled control of system clock
16:29:40.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.129.190@KRBTEST.COM; defaulting to no policy
16:29:40.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:29:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.129.190@KRBTEST.COM" created.
16:29:40.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:29:40.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.129.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab.
16:29:40.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.129.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab.
16:29:40.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.129.190@KRBTEST.COM; defaulting to no policy
16:29:40.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:29:40.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.129.190@KRBTEST.COM" created.
16:29:40.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:29:40.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.129.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab.
16:29:40.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.129.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab.
16:29:40.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:40.297394 2566 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:40.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:40.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/wal
16:29:40.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data
16:29:40.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/logs
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data/info.pb
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.129.190:46287
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.129.190
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.129.190
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.129.148:46059
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:40.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:40.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:29:40.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.129.190:46287,127.2.129.189:44571,127.2.129.188:45547
16:29:40.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:29:40.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33827/jwks.json
16:29:40.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:29:40.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:29:40.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:40.681634 2595 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:29:40.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:40.682367 2595 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:29:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:40.683408 2595 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:29:40.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:40.721614 2595 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:29:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:40.721946 2595 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:29:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:40.722404 2595 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:29:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:40.722729 2595 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:29:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:40.723099 2595 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:29:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:40.723433 2595 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:29:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:40.723701 2595 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33827/jwks.json
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:40.767496 2595 master_runner.cc:386] Master server non-default flags:
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.129.148:46059
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/wal
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.129.190:46287,127.2.129.189:44571,127.2.129.188:45547
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.129.190
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.129.190:46287
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33827/jwks.json
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data/info.pb
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.129.190
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/logs
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:29:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:29:40.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:40.768923 2595 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:29:40.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:40.770002 2595 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:29:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:40.782521 2601 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:42.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:42.188555 2600 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 2595
16:29:42.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:42.576563 2600 kernel_stack_watchdog.cc:198] Thread 2595 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
16:29:42.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:29:42.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:29:42.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:29:42.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:29:42.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:42.577306 2595 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.794s user 0.657s sys 1.137s
16:29:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:42.577801 2595 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.794s user 0.657s sys 1.137s
16:29:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:42.578071 2603 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1791 milliseconds
16:29:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:40.784570 2602 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:42.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.581107 2595 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:29:42.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:42.581139 2605 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:42.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.586656 2595 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:29:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.589874 2595 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:29:42.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.591308 2595 hybrid_clock.cc:648] HybridClock initialized: now 1773764982591279 us; error 65 us; skew 500 ppm
16:29:42.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.592378 2595 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:29:42.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:42 dist-test-slave-jdhl krb5kdc[2577](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764982, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:29:42.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.600128 2595 init.cc:377] Logged in from keytab as kudu/127.2.129.190@KRBTEST.COM (short username kudu)
16:29:42.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.606433 2595 webserver.cc:492] Webserver started at http://127.2.129.190:34409/ using document root <none> and password file <none>
16:29:42.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.607539 2595 fs_manager.cc:362] Metadata directory not provided
16:29:42.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.607756 2595 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:29:42.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.608356 2595 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:29:42.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.613116 2595 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data/instance:
16:29:42.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25d8bf1b13a0406ba46cf4eebc045b13"
16:29:42.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:42 on dist-test-slave-jdhl"
16:29:42.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.614584 2595 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/wal/instance:
16:29:42.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25d8bf1b13a0406ba46cf4eebc045b13"
16:29:42.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:42 on dist-test-slave-jdhl"
16:29:42.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.623421 2595 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.002s
16:29:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.630729 2615 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:29:42.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.632141 2595 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
16:29:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.632622 2595 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/wal
16:29:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25d8bf1b13a0406ba46cf4eebc045b13"
16:29:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:42 on dist-test-slave-jdhl"
16:29:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.633016 2595 fs_report.cc:389] FS layout report
16:29:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:29:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/wal
16:29:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/wal
16:29:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data/data
16:29:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:29:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:29:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:29:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:29:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:29:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:29:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:29:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:29:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:29:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:29:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:29:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.715322 2595 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:29:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.728508 2595 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:29:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.729147 2595 kserver.cc:163] Server-wide thread pool size limit: 3276
16:29:42.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.846340 2595 rpc_server.cc:307] RPC server started. Bound to: 127.2.129.190:46287
16:29:42.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.846522 2666 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.129.190:46287 every 8 connection(s)
16:29:42.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.850517 2595 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data/info.pb
16:29:42.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.860293 2566 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 2595
16:29:42.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.860410 2667 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:29:42.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.860864 2566 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/wal/instance
16:29:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.129.189@KRBTEST.COM; defaulting to no policy
16:29:42.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.883188 2667 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } has no permanent_uuid. Determining permanent_uuid...
16:29:42.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:29:42.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.129.189@KRBTEST.COM" created.
16:29:42.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:42 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764982, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.190@KRBTEST.COM for kudu/127.2.129.190@KRBTEST.COM
16:29:42.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:29:42.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.129.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab.
16:29:42.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.129.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab.
16:29:42.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:42.958099 2667 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } has no permanent_uuid. Determining permanent_uuid...
16:29:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:42.961223 2616 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111)
16:29:42.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.129.189@KRBTEST.COM; defaulting to no policy
16:29:42.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:42.967541 2667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.189:44571: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111)
16:29:42.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:29:42.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.129.189@KRBTEST.COM" created.
16:29:42.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:29:42.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.129.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab.
16:29:42.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.129.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab.
16:29:43.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:43.001232 2566 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:43.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:43.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/wal
16:29:43.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data
16:29:43.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:29:43.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:29:43.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:29:43.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:43.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:43.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:43.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/logs
16:29:43.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data/info.pb
16:29:43.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:43.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:29:43.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:29:43.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:29:43.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.129.189:44571
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.129.189
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.129.189
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.129.148:46059
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.129.190:46287,127.2.129.189:44571,127.2.129.188:45547
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33827/jwks.json
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:29:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:29:43.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:43.017349 2667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } attempt: 1
16:29:43.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:43.023909 2667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.189:44571: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111)
16:29:43.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:43.092445 2667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } attempt: 2
16:29:43.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:43.098649 2667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.189:44571: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111)
16:29:43.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:43.190274 2667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } attempt: 3
16:29:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:43.196136 2667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.189:44571: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111)
16:29:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:43.339695 2667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } attempt: 4
16:29:43.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:43.344895 2667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.189:44571: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111)
16:29:43.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:43.409050 2676 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:29:43.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:43.409932 2676 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:29:43.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:43.411032 2676 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:29:43.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:43.461915 2676 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:29:43.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:43.462390 2676 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:29:43.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:43.462968 2676 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:29:43.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:43.463308 2676 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:29:43.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:43.463629 2676 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:29:43.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:43.463943 2676 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:29:43.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:43.464205 2676 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33827/jwks.json
16:29:43.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:43.515359 2676 master_runner.cc:386] Master server non-default flags:
16:29:43.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:43.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.129.148:46059
16:29:43.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:43.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:43.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data
16:29:43.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/wal
16:29:43.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:29:43.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:29:43.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.129.190:46287,127.2.129.189:44571,127.2.129.188:45547
16:29:43.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab
16:29:43.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.129.189
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.129.189:44571
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33827/jwks.json
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data/info.pb
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.129.189
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/logs
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:29:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:43.517156 2676 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:29:43.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:43.518450 2676 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:29:43.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:43.532053 2683 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:43.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:43.644464 2667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } attempt: 5
16:29:43.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:43.653616 2667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.189:44571: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111)
16:29:44.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:44.201243 2667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } attempt: 6
16:29:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:44.215387 2667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.189:44571: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111)
16:29:45.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:43.533098 2684 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:45.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:45.262295 2687 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:45.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:45.272109 2685 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1729 milliseconds
16:29:45.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.272255 2676 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:29:45.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.273914 2676 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:29:45.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.276263 2667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } attempt: 7
16:29:45.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.277990 2676 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:29:45.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.279560 2676 hybrid_clock.cc:648] HybridClock initialized: now 1773764985279506 us; error 39 us; skew 500 ppm
16:29:45.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.281011 2676 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:29:45.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:45.284679 2667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.189:44571: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111)
16:29:45.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:45 dist-test-slave-jdhl krb5kdc[2577](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764985, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:29:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.289786 2676 init.cc:377] Logged in from keytab as kudu/127.2.129.189@KRBTEST.COM (short username kudu)
16:29:45.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.295796 2676 webserver.cc:492] Webserver started at http://127.2.129.189:36995/ using document root <none> and password file <none>
16:29:45.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.296906 2676 fs_manager.cc:362] Metadata directory not provided
16:29:45.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.297192 2676 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:29:45.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.297763 2676 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:29:45.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.302718 2676 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data/instance:
16:29:45.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9d3a7751ec842c682fce82d7e5f73f6"
16:29:45.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:45 on dist-test-slave-jdhl"
16:29:45.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.304631 2676 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/wal/instance:
16:29:45.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9d3a7751ec842c682fce82d7e5f73f6"
16:29:45.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:45 on dist-test-slave-jdhl"
16:29:45.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.314595 2676 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.004s sys 0.006s
16:29:45.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.323884 2700 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:29:45.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.325897 2676 fs_manager.cc:730] Time spent opening block manager: real 0.008s user 0.008s sys 0.000s
16:29:45.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.326360 2676 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/wal
16:29:45.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9d3a7751ec842c682fce82d7e5f73f6"
16:29:45.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:45 on dist-test-slave-jdhl"
16:29:45.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.327035 2676 fs_report.cc:389] FS layout report
16:29:45.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:29:45.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/wal
16:29:45.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/wal
16:29:45.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data/data
16:29:45.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:29:45.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:29:45.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:29:45.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:29:45.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:29:45.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:29:45.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:29:45.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:29:45.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:29:45.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:29:45.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:29:45.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.388123 2676 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:29:45.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.394870 2676 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:29:45.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.395414 2676 kserver.cc:163] Server-wide thread pool size limit: 3276
16:29:45.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.483335 2676 rpc_server.cc:307] RPC server started. Bound to: 127.2.129.189:44571
16:29:45.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.483475 2751 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.129.189:44571 every 8 connection(s)
16:29:45.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.486385 2676 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data/info.pb
16:29:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.493861 2752 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:29:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.493921 2566 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 2676
16:29:45.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.494407 2566 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/wal/instance
16:29:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.129.188@KRBTEST.COM; defaulting to no policy
16:29:45.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.511080 2752 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } has no permanent_uuid. Determining permanent_uuid...
16:29:45.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:29:45.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.129.188@KRBTEST.COM" created.
16:29:45.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:45 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764985, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.189@KRBTEST.COM for kudu/127.2.129.190@KRBTEST.COM
16:29:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:29:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.129.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab.
16:29:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.129.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab.
16:29:45.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.561447 2752 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } has no permanent_uuid. Determining permanent_uuid...
16:29:45.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.129.188@KRBTEST.COM; defaulting to no policy
16:29:45.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:29:45.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.129.188@KRBTEST.COM" created.
16:29:45.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:45 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764985, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.189@KRBTEST.COM for kudu/127.2.129.189@KRBTEST.COM
16:29:45.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.594058 2752 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } has no permanent_uuid. Determining permanent_uuid...
16:29:45.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:45.597581 2703 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.129.188:45547: connect: Connection refused (error 111)
16:29:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:45.602387 2752 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.188:45547: Network error: Client connection negotiation failed: client connection to 127.2.129.188:45547: connect: Connection refused (error 111)
16:29:45.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:29:45.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.129.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab.
16:29:45.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.129.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab.
16:29:45.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.609449 2566 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/wal
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/logs
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data/info.pb
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.129.188:45547
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.129.188
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.129.188
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.129.148:46059
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.129.190:46287,127.2.129.189:44571,127.2.129.188:45547
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33827/jwks.json
16:29:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:29:45.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:29:45.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.652288 2752 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } attempt: 1
16:29:45.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:45.657402 2752 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.188:45547: Network error: Client connection negotiation failed: client connection to 127.2.129.188:45547: connect: Connection refused (error 111)
16:29:45.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.725961 2752 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } attempt: 2
16:29:45.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:45.731531 2752 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.188:45547: Network error: Client connection negotiation failed: client connection to 127.2.129.188:45547: connect: Connection refused (error 111)
16:29:45.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.823020 2752 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } attempt: 3
16:29:45.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:45.827831 2752 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.188:45547: Network error: Client connection negotiation failed: client connection to 127.2.129.188:45547: connect: Connection refused (error 111)
16:29:45.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:45.971252 2752 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } attempt: 4
16:29:45.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:45.975795 2752 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.188:45547: Network error: Client connection negotiation failed: client connection to 127.2.129.188:45547: connect: Connection refused (error 111)
16:29:45.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:45.992623 2762 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:29:45.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:45.993360 2762 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:29:45.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:45.994236 2762 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:29:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:46.030684 2762 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:29:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:46.031023 2762 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:29:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:46.031453 2762 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:29:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:46.031772 2762 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:29:46.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:46.032095 2762 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:29:46.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:46.032400 2762 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:29:46.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:46.032680 2762 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33827/jwks.json
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:46.073992 2762 master_runner.cc:386] Master server non-default flags:
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.129.148:46059
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/wal
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.129.190:46287,127.2.129.189:44571,127.2.129.188:45547
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.129.188
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.129.188:45547
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:29:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33827/jwks.json
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data/info.pb
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.129.188
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/logs
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:29:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:29:46.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:46.075768 2762 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:29:46.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:46.076930 2762 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:29:46.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:46.090310 2769 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:46.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:46.275228 2752 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } attempt: 5
16:29:46.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:46.279539 2752 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.188:45547: Network error: Client connection negotiation failed: client connection to 127.2.129.188:45547: connect: Connection refused (error 111)
16:29:46.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:46.826977 2752 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } attempt: 6
16:29:46.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:46.834672 2752 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.188:45547: Network error: Client connection negotiation failed: client connection to 127.2.129.188:45547: connect: Connection refused (error 111)
16:29:47.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.375237 2667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } attempt: 8
16:29:47.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:47 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764982, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.190@KRBTEST.COM for kudu/127.2.129.189@KRBTEST.COM
16:29:47.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.404846 2667 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } has no permanent_uuid. Determining permanent_uuid...
16:29:47.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:47.412611 2667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.188:45547: Network error: Client connection negotiation failed: client connection to 127.2.129.188:45547: connect: Connection refused (error 111)
16:29:47.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.478098 2667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } attempt: 1
16:29:47.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:47.482887 2667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.188:45547: Network error: Client connection negotiation failed: client connection to 127.2.129.188:45547: connect: Connection refused (error 111)
16:29:47.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:46.097299 2773 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:47.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:46.091710 2770 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:47.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:47.483966 2771 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
16:29:47.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.484097 2762 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:29:47.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.492097 2762 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:29:47.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.495806 2762 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:29:47.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.497311 2762 hybrid_clock.cc:648] HybridClock initialized: now 1773764987497282 us; error 70 us; skew 500 ppm
16:29:47.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.498200 2762 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:29:47.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:47 dist-test-slave-jdhl krb5kdc[2577](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764987, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:29:47.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.506150 2762 init.cc:377] Logged in from keytab as kudu/127.2.129.188@KRBTEST.COM (short username kudu)
16:29:47.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.513091 2762 webserver.cc:492] Webserver started at http://127.2.129.188:44951/ using document root <none> and password file <none>
16:29:47.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.514242 2762 fs_manager.cc:362] Metadata directory not provided
16:29:47.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.514545 2762 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:29:47.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.515143 2762 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
16:29:47.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.519817 2762 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data/instance:
16:29:47.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84bd0f13a4024b3bac729dee187bce29"
16:29:47.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:47 on dist-test-slave-jdhl"
16:29:47.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.521306 2762 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/wal/instance:
16:29:47.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84bd0f13a4024b3bac729dee187bce29"
16:29:47.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:47 on dist-test-slave-jdhl"
16:29:47.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.530010 2762 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.010s sys 0.000s
16:29:47.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.536440 2667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } attempt: 2
16:29:47.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.538141 2787 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:29:47.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.540030 2762 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.006s sys 0.001s
16:29:47.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.540511 2762 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/wal
16:29:47.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84bd0f13a4024b3bac729dee187bce29"
16:29:47.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:47 on dist-test-slave-jdhl"
16:29:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.541303 2762 fs_report.cc:389] FS layout report
16:29:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:29:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/wal
16:29:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/wal
16:29:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data/data
16:29:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:29:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:29:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:29:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:29:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:29:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:29:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:29:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:29:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:29:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:29:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:29:47.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:47.542981 2667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.188:45547: Network error: Client connection negotiation failed: client connection to 127.2.129.188:45547: connect: Connection refused (error 111)
16:29:47.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.608376 2762 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:29:47.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.617518 2762 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:29:47.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.618005 2762 kserver.cc:163] Server-wide thread pool size limit: 3276
16:29:47.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.619462 2667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } attempt: 3
16:29:47.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:47.625247 2667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.129.188:45547: Network error: Client connection negotiation failed: client connection to 127.2.129.188:45547: connect: Connection refused (error 111)
16:29:47.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.727712 2762 rpc_server.cc:307] RPC server started. Bound to: 127.2.129.188:45547
16:29:47.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.727850 2838 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.129.188:45547 every 8 connection(s)
16:29:47.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.730679 2762 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data/info.pb
16:29:47.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.732928 2566 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 2762
16:29:47.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.733812 2566 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/wal/instance
16:29:47.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.739799 2839 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:29:47.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.757493 2839 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } has no permanent_uuid. Determining permanent_uuid...
16:29:47.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.780719 2667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } attempt: 4
16:29:47.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:47 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764987, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.188@KRBTEST.COM for kudu/127.2.129.190@KRBTEST.COM
16:29:47.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:47 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764980, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.129.188@KRBTEST.COM
16:29:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:47 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764982, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.190@KRBTEST.COM for kudu/127.2.129.188@KRBTEST.COM
16:29:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.808739 2839 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } has no permanent_uuid. Determining permanent_uuid...
16:29:47.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:47 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764987, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.188@KRBTEST.COM for kudu/127.2.129.189@KRBTEST.COM
16:29:47.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:47 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764980, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.129.189@KRBTEST.COM
16:29:47.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.844671 2667 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: Bootstrap starting.
16:29:47.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.860924 2667 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: Neither blocks nor log segments found. Creating new log.
16:29:47.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.870431 2667 log.cc:826] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: Log is configured to *not* fsync() on all Append() calls
16:29:47.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.871425 2839 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } has no permanent_uuid. Determining permanent_uuid...
16:29:47.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:47 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764987, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.188@KRBTEST.COM for kudu/127.2.129.188@KRBTEST.COM
16:29:47.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.895354 2752 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } attempt: 7
16:29:47.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.895869 2667 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: No bootstrap required, opened a new log
16:29:47.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:47 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764980, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.129.190@KRBTEST.COM
16:29:47.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.931520 2839 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: Bootstrap starting.
16:29:47.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.934108 2566 external_mini_cluster.cc:949] 0 TS(s) registered with all masters
16:29:47.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:47 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764985, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.189@KRBTEST.COM for kudu/127.2.129.188@KRBTEST.COM
16:29:47.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.942179 2839 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: Neither blocks nor log segments found. Creating new log.
16:29:47.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.941699 2667 raft_consensus.cc:359] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:47.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.942855 2667 raft_consensus.cc:385] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:29:47.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.943202 2667 raft_consensus.cc:740] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 25d8bf1b13a0406ba46cf4eebc045b13, State: Initialized, Role: FOLLOWER
16:29:47.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.944700 2667 consensus_queue.cc:260] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:47.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.946249 2839 log.cc:826] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: Log is configured to *not* fsync() on all Append() calls
16:29:47.952 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:29:47.952 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
16:29:47.955 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
16:29:47.955 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
16:29:47.957 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.2.129.190"
port: 46287
}
bound_http_address {
host: "127.2.129.190"
port: 34409
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.2.129.189"
port: 44571
}
bound_http_address {
host: "127.2.129.189"
port: 36995
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.2.129.188"
port: 45547
}
bound_http_address {
host: "127.2.129.188"
port: 44951
}
}
}
16:29:47.958 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:29:47.959 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:29:47.959 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:29:47.959 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:29:47.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.966440 2849 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:29:47.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.967437 2849 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: This master's current role is: FOLLOWER
16:29:47.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.969421 2667 sys_catalog.cc:565] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: configured and running, proceeding with master startup.
16:29:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.976960 2839 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: No bootstrap required, opened a new log
16:29:47.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.982379 2752 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6: Bootstrap starting.
16:29:47.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:47.995472 2621 tablet.cc:2379] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:29:47.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:47.999435 2752 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6: Neither blocks nor log segments found. Creating new log.
16:29:48.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.003811 2752 log.cc:826] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6: Log is configured to *not* fsync() on all Append() calls
16:29:48.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.015741 2839 raft_consensus.cc:359] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:48.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.016896 2839 raft_consensus.cc:385] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:29:48.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.017271 2839 raft_consensus.cc:740] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84bd0f13a4024b3bac729dee187bce29, State: Initialized, Role: FOLLOWER
16:29:48.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.018846 2839 consensus_queue.cc:260] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.029408 2752 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6: No bootstrap required, opened a new log
16:29:48.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:48 dist-test-slave-jdhl krb5kdc[2577](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764988, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:29:48.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:29:48.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.042022 2860 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:29:48.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.043354 2860 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: This master's current role is: FOLLOWER
16:29:48.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.046655 2839 sys_catalog.cc:565] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: configured and running, proceeding with master startup.
16:29:48.048 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:29:48.050 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5cc
16:29:48.057 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:29:48.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.084005 2752 raft_consensus.cc:359] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:48.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.088837 2752 raft_consensus.cc:385] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:29:48.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.090286 2752 raft_consensus.cc:740] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b9d3a7751ec842c682fce82d7e5f73f6, State: Initialized, Role: FOLLOWER
16:29:48.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.091687 2752 consensus_queue.cc:260] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:48.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:48.092604 2866 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:29:48.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:48.092983 2866 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:29:48.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.102154 2872 sys_catalog.cc:455] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:29:48.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.106887 2872 sys_catalog.cc:458] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [sys.catalog]: This master's current role is: FOLLOWER
16:29:48.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.106442 2752 sys_catalog.cc:565] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [sys.catalog]: configured and running, proceeding with master startup.
16:29:48.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:48.125695 2793 tablet.cc:2379] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:29:48.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.085470 2595 master_runner.cc:186] Error getting master registration for 127.2.129.189:44571: OK, instance_id { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" instance_seqno: 1773764985457037 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:29:48.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.135589 2595 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:29:48.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:48.154162 2706 tablet.cc:2379] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:29:48.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:48.156901 2882 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:29:48.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:48.157481 2882 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:29:48.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.163874 2762 master_runner.cc:186] Error getting master registration for 127.2.129.189:44571: OK, instance_id { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" instance_seqno: 1773764985457037 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:29:48.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.201074 2762 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:29:48.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:48.232721 2890 catalog_manager.cc:1568] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:29:48.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:48.233170 2890 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:29:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.239805 2676 master_runner.cc:263] No leader master found from master b9d3a7751ec842c682fce82d7e5f73f6
16:29:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.240191 2676 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:29:48.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.357426 2849 raft_consensus.cc:493] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:29:48.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.358078 2849 raft_consensus.cc:515] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:48.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.362699 2727 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" is_pre_election: true
16:29:48.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.363636 2727 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 25d8bf1b13a0406ba46cf4eebc045b13 in term 0.
16:29:48.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.367153 2616 leader_election.cc:304] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 25d8bf1b13a0406ba46cf4eebc045b13, b9d3a7751ec842c682fce82d7e5f73f6; no voters:
16:29:48.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.370620 2814 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84bd0f13a4024b3bac729dee187bce29" is_pre_election: true
16:29:48.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.371730 2814 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 25d8bf1b13a0406ba46cf4eebc045b13 in term 0.
16:29:48.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.373622 2849 leader_election.cc:290] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b9d3a7751ec842c682fce82d7e5f73f6 (127.2.129.189:44571), 84bd0f13a4024b3bac729dee187bce29 (127.2.129.188:45547)
16:29:48.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.374301 2849 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 0 FOLLOWER]: Leader pre-election won for term 1
16:29:48.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.374691 2849 raft_consensus.cc:493] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:29:48.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.375029 2849 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 0 FOLLOWER]: Advancing to term 1
16:29:48.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.384908 2849 raft_consensus.cc:515] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:48.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.387502 2849 leader_election.cc:290] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 1 election: Requested vote from peers b9d3a7751ec842c682fce82d7e5f73f6 (127.2.129.189:44571), 84bd0f13a4024b3bac729dee187bce29 (127.2.129.188:45547)
16:29:48.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.390470 2727 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b9d3a7751ec842c682fce82d7e5f73f6"
16:29:48.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.391165 2814 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84bd0f13a4024b3bac729dee187bce29"
16:29:48.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.391405 2727 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [term 0 FOLLOWER]: Advancing to term 1
16:29:48.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.391753 2814 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 0 FOLLOWER]: Advancing to term 1
16:29:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.399446 2727 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 25d8bf1b13a0406ba46cf4eebc045b13 in term 1.
16:29:48.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.399619 2814 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 25d8bf1b13a0406ba46cf4eebc045b13 in term 1.
16:29:48.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.401158 2618 leader_election.cc:304] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 25d8bf1b13a0406ba46cf4eebc045b13, 84bd0f13a4024b3bac729dee187bce29; no voters:
16:29:48.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.402138 2849 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 1 FOLLOWER]: Leader election won for term 1
16:29:48.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.402629 2849 raft_consensus.cc:697] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 1 LEADER]: Becoming Leader. State: Replica: 25d8bf1b13a0406ba46cf4eebc045b13, State: Running, Role: LEADER
16:29:48.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.403674 2849 consensus_queue.cc:237] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:48.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.425305 2892 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 25d8bf1b13a0406ba46cf4eebc045b13. Latest consensus state: current_term: 1 leader_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:29:48.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.427088 2892 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: This master's current role is: LEADER
16:29:48.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.436276 2894 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
16:29:48.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.444283 2894 catalog_manager.cc:1494] Initializing Kudu cluster ID...
16:29:48.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.473495 2814 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 1 FOLLOWER]: Refusing update from remote peer 25d8bf1b13a0406ba46cf4eebc045b13: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:29:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.479948 2727 raft_consensus.cc:1275] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [term 1 FOLLOWER]: Refusing update from remote peer 25d8bf1b13a0406ba46cf4eebc045b13: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:29:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.483196 2849 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:29:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.484895 2849 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:29:48.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.536829 2860 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 25d8bf1b13a0406ba46cf4eebc045b13. Latest consensus state: current_term: 1 leader_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:29:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.538019 2860 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: This master's current role is: FOLLOWER
16:29:48.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.564819 2896 mvcc.cc:204] Tried to move back new op lower bound from 7265341392761413632 to 7265341392518156288. Current Snapshot: MvccSnapshot[applied={T|T < 7265341392761413632}]
16:29:48.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.578606 2872 sys_catalog.cc:455] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 25d8bf1b13a0406ba46cf4eebc045b13. Latest consensus state: current_term: 1 leader_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:29:48.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.580077 2872 sys_catalog.cc:458] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [sys.catalog]: This master's current role is: FOLLOWER
16:29:48.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.589493 2849 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:29:48.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.590342 2849 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: This master's current role is: LEADER
16:29:48.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.599608 2849 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:29:48.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.600435 2849 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: This master's current role is: LEADER
16:29:48.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.602146 2860 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:29:48.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.603587 2860 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: This master's current role is: FOLLOWER
16:29:48.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.614708 2872 sys_catalog.cc:455] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:29:48.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.615556 2872 sys_catalog.cc:458] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [sys.catalog]: This master's current role is: FOLLOWER
16:29:48.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.619402 2894 catalog_manager.cc:1357] Generated new cluster ID: d116fbc0cf314826bec437e0b8601c6f
16:29:48.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.619789 2894 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
16:29:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.676455 2894 catalog_manager.cc:1380] Generated new certificate authority record
16:29:48.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.696136 2894 catalog_manager.cc:1514] Loading token signing keys...
16:29:48.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.772545 2894 catalog_manager.cc:6044] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: Generated new TSK 0
16:29:48.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:48.773819 2894 catalog_manager.cc:1524] Initializing in-progress tserver states...
16:29:49.062 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.129.190:46287
16:29:49.062 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
16:29:49.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:49.063241 2566 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 2595
16:29:49.098 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:29:49.098 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.129.188:45547
16:29:49.098 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
16:29:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:49.099100 2566 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 2762
16:29:49.133 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:29:49.133 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.129.189:44571
16:29:49.134 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
16:29:49.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:49.134572 2566 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 2676
16:29:49.171 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:29:49.171 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.129.190:46287
16:29:49.171 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
16:29:49.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:49.173906 2566 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:49.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:49.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/wal
16:29:49.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data
16:29:49.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:29:49.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:29:49.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:29:49.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:49.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:49.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:49.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/logs
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data/info.pb
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.129.190:46287
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.129.190
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34409
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.129.190
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.129.148:46059
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.129.190:46287,127.2.129.189:44571,127.2.129.188:45547
16:29:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:29:49.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33827/jwks.json
16:29:49.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:29:49.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:29:49.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:49.559257 2904 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:29:49.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:49.559836 2904 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:29:49.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:49.560510 2904 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:29:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:49.595014 2904 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:29:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:49.595253 2904 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:29:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:49.595644 2904 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:29:49.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:49.595964 2904 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:29:49.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:49.596186 2904 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:29:49.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:49.596390 2904 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:29:49.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:49.596618 2904 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33827/jwks.json
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:49.638797 2904 master_runner.cc:386] Master server non-default flags:
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.129.148:46059
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/wal
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.129.190:46287,127.2.129.189:44571,127.2.129.188:45547
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.129.190
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.129.190:46287
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33827/jwks.json
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data/info.pb
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.129.190
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34409
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/logs
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:29:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:29:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:29:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:29:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:29:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:29:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:29:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:49.640149 2904 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:29:49.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:49.641188 2904 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:29:49.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:49.653855 2910 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:51.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:49.654018 2911 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:51.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:51.063256 2909 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 2904
16:29:51.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:51.680411 2904 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.027s user 0.721s sys 1.224s
16:29:51.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:51.684257 2904 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.031s user 0.721s sys 1.224s
16:29:51.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:51.685089 2913 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:51.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:51.691921 2912 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2036 milliseconds
16:29:51.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.692529 2904 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:29:51.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.693679 2904 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:29:51.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.696508 2904 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:29:51.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.698372 2904 hybrid_clock.cc:648] HybridClock initialized: now 1773764991698280 us; error 105 us; skew 500 ppm
16:29:51.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.699604 2904 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:29:51.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:51 dist-test-slave-jdhl krb5kdc[2577](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764991, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:29:51.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.709914 2904 init.cc:377] Logged in from keytab as kudu/127.2.129.190@KRBTEST.COM (short username kudu)
16:29:51.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.717190 2904 webserver.cc:492] Webserver started at http://127.2.129.190:34409/ using document root <none> and password file <none>
16:29:51.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.718323 2904 fs_manager.cc:362] Metadata directory not provided
16:29:51.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.719205 2904 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:29:51.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.733680 2904 fs_manager.cc:714] Time spent opening directory manager: real 0.009s user 0.008s sys 0.000s
16:29:51.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.743232 2923 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:29:51.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.744784 2904 fs_manager.cc:730] Time spent opening block manager: real 0.007s user 0.003s sys 0.004s
16:29:51.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.745142 2904 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/wal
16:29:51.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25d8bf1b13a0406ba46cf4eebc045b13"
16:29:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:42 on dist-test-slave-jdhl"
16:29:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.747584 2904 fs_report.cc:389] FS layout report
16:29:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:29:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/wal
16:29:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/wal
16:29:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data/data
16:29:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:29:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:29:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:29:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:29:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:29:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:29:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:29:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:29:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:29:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:29:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:29:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.815889 2904 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:29:51.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.825243 2904 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:29:51.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.826246 2904 kserver.cc:163] Server-wide thread pool size limit: 3276
16:29:51.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.913080 2904 rpc_server.cc:307] RPC server started. Bound to: 127.2.129.190:46287
16:29:51.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.913166 2975 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.129.190:46287 every 8 connection(s)
16:29:51.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.916364 2904 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data/info.pb
16:29:51.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.920033 2566 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 2904
16:29:51.920 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:29:51.921 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.129.188:45547
16:29:51.921 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.923794 2566 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/wal
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/logs
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data/info.pb
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.129.188:45547
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.129.188
16:29:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44951
16:29:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab
16:29:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.129.188
16:29:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:29:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:29:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:29:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:29:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.129.148:46059
16:29:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:29:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.129.190:46287,127.2.129.189:44571,127.2.129.188:45547
16:29:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:29:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33827/jwks.json
16:29:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:29:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:29:51.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.930702 2976 sys_catalog.cc:263] Verifying existing consensus state
16:29:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.941262 2976 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: Bootstrap starting.
16:29:51.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:51.996852 2976 log.cc:826] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: Log is configured to *not* fsync() on all Append() calls
16:29:52.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:52.021181 2976 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:29:52.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:52.022312 2976 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: Bootstrap complete.
16:29:52.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:52.051676 2976 raft_consensus.cc:359] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:52.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:52.054352 2976 raft_consensus.cc:740] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 25d8bf1b13a0406ba46cf4eebc045b13, State: Initialized, Role: FOLLOWER
16:29:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:52.055370 2976 consensus_queue.cc:260] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:52.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:52.058862 2980 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:29:52.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:52.059806 2980 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: This master's current role is: FOLLOWER
16:29:52.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:52.060796 2976 sys_catalog.cc:565] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: configured and running, proceeding with master startup.
16:29:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:52.113478 2991 catalog_manager.cc:1269] Loaded cluster ID: d116fbc0cf314826bec437e0b8601c6f
16:29:52.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:52.114722 2991 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: loading cluster ID for follower catalog manager: success
16:29:52.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:52.124799 2991 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: acquiring CA information for follower catalog manager: success
16:29:52.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:52 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764991, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.190@KRBTEST.COM for kudu/127.2.129.190@KRBTEST.COM
16:29:52.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:52.131564 2991 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:29:52.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:52.158927 2904 master_runner.cc:186] Error getting master registration for 127.2.129.189:44571: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111),
16:29:52.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:52.184330 2904 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:29:52.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:52.436198 2978 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:29:52.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:52.436784 2978 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:29:52.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:52.437464 2978 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:29:52.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:52.476789 2978 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:29:52.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:52.477034 2978 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:29:52.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:52.477303 2978 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:29:52.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:52.477514 2978 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:29:52.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:52.477710 2978 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:29:52.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:52.477901 2978 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:29:52.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:52.478047 2978 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33827/jwks.json
16:29:52.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:52.520392 2978 master_runner.cc:386] Master server non-default flags:
16:29:52.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:52.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.129.148:46059
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/wal
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.129.190:46287,127.2.129.189:44571,127.2.129.188:45547
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.129.188
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.129.188:45547
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33827/jwks.json
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data/info.pb
16:29:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.129.188
16:29:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44951
16:29:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:29:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:29:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:29:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/logs
16:29:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:29:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:29:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:29:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:29:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:29:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:29:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:29:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:29:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:29:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:52.521875 2978 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:29:52.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:52.523080 2978 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:29:52.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:52.535912 3001 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:53.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:53.759236 3010 raft_consensus.cc:493] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:29:53.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:53.763839 3010 raft_consensus.cc:515] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:53.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:53.799240 3010 leader_election.cc:290] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers b9d3a7751ec842c682fce82d7e5f73f6 (127.2.129.189:44571), 84bd0f13a4024b3bac729dee187bce29 (127.2.129.188:45547)
16:29:53.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:53.816722 2926 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.129.188:45547: connect: Connection refused (error 111)
16:29:53.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:53.836846 2926 leader_election.cc:336] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 84bd0f13a4024b3bac729dee187bce29 (127.2.129.188:45547): Network error: Client connection negotiation failed: client connection to 127.2.129.188:45547: connect: Connection refused (error 111)
16:29:53.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:53.842021 2924 leader_election.cc:336] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b9d3a7751ec842c682fce82d7e5f73f6 (127.2.129.189:44571): Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111)
16:29:53.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:53.844233 2924 leader_election.cc:304] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 25d8bf1b13a0406ba46cf4eebc045b13; no voters: 84bd0f13a4024b3bac729dee187bce29, b9d3a7751ec842c682fce82d7e5f73f6
16:29:53.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:53.847267 3010 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
16:29:54.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:54.447124 2269 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8320 after lost signal to thread 2142
16:29:54.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:54.449352 2269 debug-util.cc:398] Leaking SignalData structure 0x7b08000acb60 after lost signal to thread 2272
16:29:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:53.939808 3000 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 2978
16:29:54.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:54.748589 2978 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.205s user 0.543s sys 1.151s
16:29:54.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:54.748690 3003 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2202 milliseconds
16:29:54.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:54.749029 2978 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.206s user 0.543s sys 1.151s
16:29:54.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:54.750063 2978 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:29:54.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:54.750167 3004 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:54.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:52.543427 3002 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:54.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:54.757310 2978 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:29:54.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:54.762406 2978 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:29:54.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:54.764849 2978 hybrid_clock.cc:648] HybridClock initialized: now 1773764994764742 us; error 117 us; skew 500 ppm
16:29:54.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:54.766067 2978 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:29:54.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:54 dist-test-slave-jdhl krb5kdc[2577](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764994, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:29:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:54.774296 2978 init.cc:377] Logged in from keytab as kudu/127.2.129.188@KRBTEST.COM (short username kudu)
16:29:54.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:54.780869 2978 webserver.cc:492] Webserver started at http://127.2.129.188:44951/ using document root <none> and password file <none>
16:29:54.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:54.782249 2978 fs_manager.cc:362] Metadata directory not provided
16:29:54.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:54.782613 2978 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:29:54.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:54.814031 2978 fs_manager.cc:714] Time spent opening directory manager: real 0.026s user 0.014s sys 0.003s
16:29:54.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:54.821056 3020 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:29:54.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:54.823308 2978 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.001s sys 0.005s
16:29:54.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:54.823810 2978 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/wal
16:29:54.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84bd0f13a4024b3bac729dee187bce29"
16:29:54.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:47 on dist-test-slave-jdhl"
16:29:54.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:54.826961 2978 fs_report.cc:389] FS layout report
16:29:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:29:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/wal
16:29:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/wal
16:29:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data/data
16:29:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:29:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:29:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:29:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:29:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:29:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:29:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:29:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:29:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:29:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:29:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:29:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:54.929351 2978 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:29:54.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:54.938751 2978 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:29:54.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:54.939445 2978 kserver.cc:163] Server-wide thread pool size limit: 3276
16:29:55.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.115154 2978 rpc_server.cc:307] RPC server started. Bound to: 127.2.129.188:45547
16:29:55.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.115139 3072 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.129.188:45547 every 8 connection(s)
16:29:55.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.120138 2978 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data/info.pb
16:29:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.123018 2566 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 2978
16:29:55.123 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:29:55.124 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.129.189:44571
16:29:55.124 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
16:29:55.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.125856 2566 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:55.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:29:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/wal
16:29:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data
16:29:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:29:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:29:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:29:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/logs
16:29:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data/info.pb
16:29:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.129.189:44571
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.129.189
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36995
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.129.189
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.129.148:46059
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.129.190:46287,127.2.129.189:44571,127.2.129.188:45547
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33827/jwks.json
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:29:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:29:55.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.159046 3073 sys_catalog.cc:263] Verifying existing consensus state
16:29:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.170205 3073 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: Bootstrap starting.
16:29:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.257794 3073 log.cc:826] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: Log is configured to *not* fsync() on all Append() calls
16:29:55.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.291577 3073 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:29:55.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.293164 3073 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: Bootstrap complete.
16:29:55.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.336308 3073 raft_consensus.cc:359] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:55.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.341197 3073 raft_consensus.cc:740] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84bd0f13a4024b3bac729dee187bce29, State: Initialized, Role: FOLLOWER
16:29:55.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.343668 3073 consensus_queue.cc:260] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:55.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.356667 3077 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:29:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.360765 3077 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: This master's current role is: FOLLOWER
16:29:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.360157 3073 sys_catalog.cc:565] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: configured and running, proceeding with master startup.
16:29:55.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.540331 3088 catalog_manager.cc:1269] Loaded cluster ID: d116fbc0cf314826bec437e0b8601c6f
16:29:55.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.540690 3088 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: loading cluster ID for follower catalog manager: success
16:29:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:55 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764994, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.188@KRBTEST.COM for kudu/127.2.129.190@KRBTEST.COM
16:29:55.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.577502 3088 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: acquiring CA information for follower catalog manager: success
16:29:55.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.593206 3088 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:29:55.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.596999 2978 master_runner.cc:186] Error getting master registration for 127.2.129.189:44571: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111),
16:29:55.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.630609 2978 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:29:55.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.632779 3093 raft_consensus.cc:493] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:29:55.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.633337 3093 raft_consensus.cc:515] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.652799 3093 leader_election.cc:290] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers b9d3a7751ec842c682fce82d7e5f73f6 (127.2.129.189:44571), 84bd0f13a4024b3bac729dee187bce29 (127.2.129.188:45547)
16:29:55.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:55.676628 2924 leader_election.cc:336] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b9d3a7751ec842c682fce82d7e5f73f6 (127.2.129.189:44571): Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111)
16:29:55.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.700917 3047 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "84bd0f13a4024b3bac729dee187bce29" is_pre_election: true
16:29:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.702111 3047 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 25d8bf1b13a0406ba46cf4eebc045b13 in term 1.
16:29:55.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.704594 2926 leader_election.cc:304] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 25d8bf1b13a0406ba46cf4eebc045b13, 84bd0f13a4024b3bac729dee187bce29; no voters: b9d3a7751ec842c682fce82d7e5f73f6
16:29:55.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.706838 3093 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 1 FOLLOWER]: Leader pre-election won for term 2
16:29:55.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.707232 3093 raft_consensus.cc:493] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:29:55.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.707628 3093 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 1 FOLLOWER]: Advancing to term 2
16:29:55.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.727203 3093 raft_consensus.cc:515] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:55.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:55.743137 2924 leader_election.cc:336] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer b9d3a7751ec842c682fce82d7e5f73f6 (127.2.129.189:44571): Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111)
16:29:55.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.745582 3047 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "84bd0f13a4024b3bac729dee187bce29"
16:29:55.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.746210 3047 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 1 FOLLOWER]: Advancing to term 2
16:29:55.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:55.752040 3074 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:29:55.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:55.752992 3074 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:29:55.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:55.754145 3074 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:29:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.759961 3047 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 25d8bf1b13a0406ba46cf4eebc045b13 in term 2.
16:29:55.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.761495 2926 leader_election.cc:304] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 25d8bf1b13a0406ba46cf4eebc045b13, 84bd0f13a4024b3bac729dee187bce29; no voters: b9d3a7751ec842c682fce82d7e5f73f6
16:29:55.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.756245 3093 leader_election.cc:290] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 2 election: Requested vote from peers b9d3a7751ec842c682fce82d7e5f73f6 (127.2.129.189:44571), 84bd0f13a4024b3bac729dee187bce29 (127.2.129.188:45547)
16:29:55.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.767688 3093 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 2 FOLLOWER]: Leader election won for term 2
16:29:55.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.768240 3093 raft_consensus.cc:697] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 2 LEADER]: Becoming Leader. State: Replica: 25d8bf1b13a0406ba46cf4eebc045b13, State: Running, Role: LEADER
16:29:55.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.769233 3093 consensus_queue.cc:237] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.803622 3102 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 25d8bf1b13a0406ba46cf4eebc045b13. Latest consensus state: current_term: 2 leader_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:29:55.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.804850 3102 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: This master's current role is: LEADER
16:29:55.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:55.820487 3074 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:29:55.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:55.820837 3074 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:29:55.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:55.821303 3074 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:29:55.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:55.821668 3074 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:29:55.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:55.822011 3074 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:29:55.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:55.822355 3074 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:29:55.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:55.822644 3074 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33827/jwks.json
16:29:55.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.824899 3105 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
16:29:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.845366 3105 catalog_manager.cc:1494] Initializing Kudu cluster ID...
16:29:55.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.862629 3105 catalog_manager.cc:1269] Loaded cluster ID: d116fbc0cf314826bec437e0b8601c6f
16:29:55.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.863245 3105 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
16:29:55.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.868914 3105 catalog_manager.cc:1514] Loading token signing keys...
16:29:55.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.873947 3105 catalog_manager.cc:6055] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: Loaded TSK: 0
16:29:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.888017 3105 catalog_manager.cc:1524] Initializing in-progress tserver states...
16:29:55.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.892127 3074 master_runner.cc:386] Master server non-default flags:
16:29:55.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:29:55.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.129.148:46059
16:29:55.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:29:55.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:29:55.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data
16:29:55.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/wal
16:29:55.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:29:55.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:29:55.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.129.190:46287,127.2.129.189:44571,127.2.129.188:45547
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.129.189
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.129.189:44571
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33827/jwks.json
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data/info.pb
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.129.189
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36995
16:29:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:29:55.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:29:55.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:29:55.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:29:55.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:29:55.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/logs
16:29:55.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:29:55.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:29:55.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:29:55.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:29:55.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:29:55.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:29:55.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:29:55.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:29:55.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:29:55.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:29:55.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.893946 3074 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:29:55.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:55.895418 3074 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:29:55.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:55.951123 3108 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:56.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:56.295806 3047 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 2 FOLLOWER]: Refusing update from remote peer 25d8bf1b13a0406ba46cf4eebc045b13: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
16:29:56.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:56.298389 3102 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
16:29:56.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:56.335101 3113 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 25d8bf1b13a0406ba46cf4eebc045b13. Latest consensus state: current_term: 2 leader_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:29:56.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:56.359140 3113 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: This master's current role is: FOLLOWER
16:29:56.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:56.365988 3115 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:29:56.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:56.366897 3115 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: This master's current role is: LEADER
16:29:56.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:56.368570 3113 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:29:56.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:56.369792 3113 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: This master's current role is: FOLLOWER
16:29:56.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:56.404904 2924 consensus_peers.cc:597] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 -> Peer b9d3a7751ec842c682fce82d7e5f73f6 (127.2.129.189:44571): Couldn't send request to peer b9d3a7751ec842c682fce82d7e5f73f6. Status: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
16:29:57.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:57.192708 2904 master_runner.cc:186] Error getting master registration for 127.2.129.189:44571: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111),
16:29:57.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:57.193208 2904 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:29:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:59.026140 2924 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111) [suppressed 8 similar messages]
16:29:59.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:59.032382 2924 consensus_peers.cc:597] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 -> Peer b9d3a7751ec842c682fce82d7e5f73f6 (127.2.129.189:44571): Couldn't send request to peer b9d3a7751ec842c682fce82d7e5f73f6. Status: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
16:29:59.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:57.339231 3106 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 3074
16:29:59.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:59.330413 3106 kernel_stack_watchdog.cc:198] Thread 3074 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 382ms:
16:29:59.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:29:59.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:29:59.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:29:59.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:29:59.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:55.953661 3107 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:59.333483 3074 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.378s user 0.000s sys 0.002s
16:29:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:59.334286 3109 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3378 milliseconds
16:29:59.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:59.334385 3074 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.379s user 0.000s sys 0.003s
16:29:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:29:59.339075 3111 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:29:59.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.339007 3074 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:29:59.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.352622 3074 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:29:59.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.367790 3074 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:29:59.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.371493 3074 hybrid_clock.cc:648] HybridClock initialized: now 1773764999371360 us; error 122 us; skew 500 ppm
16:29:59.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.373275 3074 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:29:59.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:59 dist-test-slave-jdhl krb5kdc[2577](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764999, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:29:59.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.382920 3074 init.cc:377] Logged in from keytab as kudu/127.2.129.189@KRBTEST.COM (short username kudu)
16:29:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.400072 3074 webserver.cc:492] Webserver started at http://127.2.129.189:36995/ using document root <none> and password file <none>
16:29:59.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.402397 3074 fs_manager.cc:362] Metadata directory not provided
16:29:59.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.403151 3074 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:29:59.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.416721 3074 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.007s sys 0.001s
16:29:59.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.423556 3137 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:29:59.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.425609 3074 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
16:29:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.426262 3074 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/wal
16:29:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9d3a7751ec842c682fce82d7e5f73f6"
16:29:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:45 on dist-test-slave-jdhl"
16:29:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.429389 3074 fs_report.cc:389] FS layout report
16:29:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:29:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/wal
16:29:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/wal
16:29:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data/data
16:29:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:29:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:29:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:29:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:29:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:29:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:29:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:29:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:29:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:29:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:29:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:29:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.501834 3074 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:29:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.513229 3074 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:29:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.513698 3074 kserver.cc:163] Server-wide thread pool size limit: 3276
16:29:59.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.631974 3074 rpc_server.cc:307] RPC server started. Bound to: 127.2.129.189:44571
16:29:59.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.632052 3189 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.129.189:44571 every 8 connection(s)
16:29:59.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.635685 3074 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data/info.pb
16:29:59.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.645864 2566 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 3074
16:29:59.646 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:29:59.648 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.129.190:46287(127.2.129.190:46287)] connecting to peer
16:29:59.649 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.129.188:45547(127.2.129.188:45547)] connecting to peer
16:29:59.650 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.2.129.190:46287(127.2.129.190:46287)] Successfully connected to peer
16:29:59.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.650363 3190 sys_catalog.cc:263] Verifying existing consensus state
16:29:59.652 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.129.189:44571(127.2.129.189:44571)] connecting to peer
16:29:59.653 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.2.129.188:45547(127.2.129.188:45547)] Successfully connected to peer
16:29:59.654 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.2.129.189:44571(127.2.129.189:44571)] Successfully connected to peer
16:29:59.657 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
16:29:59.658 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5cc
16:29:59.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:59 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773764988, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:29:59.667 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:29:59.668 [INFO - kudu-nio-0] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
16:29:59.669 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.129.190
16:29:59.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.667279 3190 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6: Bootstrap starting.
16:29:59.672 [DEBUG - kudu-nio-1] (SecurityContext.java:222) Refreshing Kerberos credentials...
16:29:59.673 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5cc
16:29:59.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:59 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773764988, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:29:59.679 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:29:59.679 [INFO - kudu-nio-1] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
16:29:59.679 [DEBUG - kudu-nio-2] (SecurityContext.java:222) Refreshing Kerberos credentials...
16:29:59.680 [DEBUG - kudu-nio-2] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5cc
16:29:59.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:59 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773764988, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:29:59.687 [DEBUG - kudu-nio-2] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:29:59.687 [INFO - kudu-nio-2] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
16:29:59.690 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.129.188
16:29:59.699 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.129.189
16:29:59.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:59 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773764988, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.129.188@KRBTEST.COM
16:29:59.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:59 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773764988, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.129.190@KRBTEST.COM
16:29:59.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.735675 3190 log.cc:826] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6: Log is configured to *not* fsync() on all Append() calls
16:29:59.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:59 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773764988, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.129.189@KRBTEST.COM
16:29:59.760 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x916d6bb6, L:/127.0.0.1:41800 - R:/127.2.129.188:45547] using SASL/GSSAPI
16:29:59.779 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x540d5066, L:/127.0.0.1:44806 - R:/127.2.129.190:46287] using SASL/GSSAPI
16:29:59.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.780893 3190 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:29:59.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.782116 3190 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6: Bootstrap complete.
16:29:59.795 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x9f23a9e8, L:/127.0.0.1:47932 - R:/127.2.129.189:44571] using SASL/GSSAPI
16:29:59.802 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
16:29:59.803 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.2.129.189:44571: Catalog manager is not initialized. State: Starting
16:29:59.803 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=156), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.129.190:46287, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.129.188:45547, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.129.189:44571, [120ms] Sub RPC ConnectToMaster: received response from server master-127.2.129.188:45547: OK, [154ms] Sub RPC ConnectToMaster: received response from server master-127.2.129.190:46287: OK, [155ms] Sub RPC ConnectToMaster: received response from server master-127.2.129.189:44571: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), deferred=null) after lookup
16:29:59.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.827214 3190 raft_consensus.cc:359] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:59.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.831559 3190 raft_consensus.cc:740] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: b9d3a7751ec842c682fce82d7e5f73f6, State: Initialized, Role: FOLLOWER
16:29:59.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.833603 3190 consensus_queue.cc:260] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:29:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.841427 3201 sys_catalog.cc:455] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:29:59.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.842725 3201 sys_catalog.cc:458] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [sys.catalog]: This master's current role is: FOLLOWER
16:29:59.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.846199 3190 sys_catalog.cc:565] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [sys.catalog]: configured and running, proceeding with master startup.
16:29:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.910466 3212 catalog_manager.cc:1269] Loaded cluster ID: d116fbc0cf314826bec437e0b8601c6f
16:29:59.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.911549 3212 catalog_manager.cc:1562] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6: loading cluster ID for follower catalog manager: success
16:29:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:29:59 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773764999, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.189@KRBTEST.COM for kudu/127.2.129.190@KRBTEST.COM
16:29:59.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.929788 3212 catalog_manager.cc:1584] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6: acquiring CA information for follower catalog manager: success
16:29:59.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:29:59.936339 3212 catalog_manager.cc:1612] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:30:00.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:00.076705 3165 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [term 1 FOLLOWER]: Advancing to term 2
16:30:00.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:00.104125 3220 sys_catalog.cc:455] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:30:00.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:00.108139 3220 sys_catalog.cc:458] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [sys.catalog]: This master's current role is: FOLLOWER
16:30:00.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:00.114720 3201 sys_catalog.cc:455] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 25d8bf1b13a0406ba46cf4eebc045b13. Latest consensus state: current_term: 2 leader_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:30:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:00.115834 3201 sys_catalog.cc:458] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6 [sys.catalog]: This master's current role is: FOLLOWER
16:30:00.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:00.599581 3121 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:30:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:00.601033 3121 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: This master's current role is: LEADER
16:30:00.809 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.129.190:46287
16:30:00.809 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
16:30:00.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:00.810269 2566 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 2904
16:30:00.857 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.2.129.190:46287(127.2.129.190:46287)] handling channelInactive
16:30:00.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.129.190:46287(127.2.129.190:46287)] cleaning up while in state READY due to: connection closed
16:30:00.858 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:30:00.858 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.129.188:45547
16:30:00.858 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
16:30:00.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:00.859077 2566 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 2978
16:30:00.931 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.2.129.188:45547(127.2.129.188:45547)] handling channelInactive
16:30:00.931 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.129.188:45547(127.2.129.188:45547)] cleaning up while in state READY due to: connection closed
16:30:00.932 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:30:00.932 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.129.189:44571
16:30:00.932 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
16:30:00.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:00.933132 2566 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 3074
16:30:00.980 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:30:00.980 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.129.190:46287
16:30:00.980 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.2.129.189:44571(127.2.129.189:44571)] handling channelInactive
16:30:00.980 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
16:30:00.980 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.2.129.189:44571(127.2.129.189:44571)] cleaning up while in state READY due to: connection closed
16:30:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:00.982091 2566 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:30:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:30:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/wal
16:30:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data
16:30:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:30:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:30:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:30:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/logs
16:30:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data/info.pb
16:30:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.129.190:46287
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.129.190
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34409
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.129.190
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:30:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:30:00.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:30:00.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.129.148:46059
16:30:00.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:00.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:00.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:00.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:00.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.129.190:46287,127.2.129.189:44571,127.2.129.188:45547
16:30:00.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:00.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33827/jwks.json
16:30:00.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:30:00.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:30:01.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:01.621105 3229 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:30:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:01.621692 3229 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:30:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:01.622491 3229 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:30:01.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:01.669040 3229 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:30:01.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:01.669333 3229 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:30:01.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:01.669616 3229 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:30:01.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:01.669982 3229 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:30:01.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:01.670411 3229 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:30:01.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:01.670825 3229 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:30:01.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:01.671139 3229 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33827/jwks.json
16:30:01.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:01.723201 3229 master_runner.cc:386] Master server non-default flags:
16:30:01.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.129.148:46059
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/wal
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.129.190:46287,127.2.129.189:44571,127.2.129.188:45547
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.129.190
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.129.190:46287
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33827/jwks.json
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data/info.pb
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.129.190
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34409
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/logs
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:30:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:30:01.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:01.725215 3229 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:30:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:01.727550 3229 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:30:01.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:01.743140 3236 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:03.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:01.752760 3239 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:03.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:01.744817 3237 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:03.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:03.640074 3238 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1878 milliseconds
16:30:03.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.640411 3229 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:30:03.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.642320 3229 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:30:03.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.646387 3229 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:30:03.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.648864 3229 hybrid_clock.cc:648] HybridClock initialized: now 1773765003649103 us; error 378 us; skew 500 ppm
16:30:03.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.649927 3229 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:30:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:30:03 dist-test-slave-jdhl krb5kdc[2577](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773765003, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:30:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.657991 3229 init.cc:377] Logged in from keytab as kudu/127.2.129.190@KRBTEST.COM (short username kudu)
16:30:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.664706 3229 webserver.cc:492] Webserver started at http://127.2.129.190:34409/ using document root <none> and password file <none>
16:30:03.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.666145 3229 fs_manager.cc:362] Metadata directory not provided
16:30:03.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.666453 3229 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:30:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.681325 3229 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.004s sys 0.004s
16:30:03.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.690214 3250 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:30:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.692196 3229 fs_manager.cc:730] Time spent opening block manager: real 0.008s user 0.002s sys 0.004s
16:30:03.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.692655 3229 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/wal
16:30:03.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25d8bf1b13a0406ba46cf4eebc045b13"
16:30:03.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:42 on dist-test-slave-jdhl"
16:30:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.695823 3229 fs_report.cc:389] FS layout report
16:30:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:30:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/wal
16:30:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/wal
16:30:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data/data
16:30:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:30:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:30:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:30:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:30:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:30:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:30:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:30:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:30:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:30:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:30:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:30:03.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.761839 3229 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:30:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.770735 3229 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:30:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.771488 3229 kserver.cc:163] Server-wide thread pool size limit: 3276
16:30:03.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.894289 3229 rpc_server.cc:307] RPC server started. Bound to: 127.2.129.190:46287
16:30:03.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.894348 3301 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.129.190:46287 every 8 connection(s)
16:30:03.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.897802 3229 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-0/data/info.pb
16:30:03.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.913305 2566 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 3229
16:30:03.914 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:30:03.914 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.129.188:45547
16:30:03.914 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
16:30:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.916132 2566 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:30:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:30:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/wal
16:30:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data
16:30:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:30:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:30:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:30:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:03.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/logs
16:30:03.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data/info.pb
16:30:03.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:03.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.129.188:45547
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.129.188
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44951
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.129.188
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.129.148:46059
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.129.190:46287,127.2.129.189:44571,127.2.129.188:45547
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33827/jwks.json
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:30:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:30:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.921602 3302 sys_catalog.cc:263] Verifying existing consensus state
16:30:03.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.934043 3302 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: Bootstrap starting.
16:30:03.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:03.996147 3302 log.cc:826] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: Log is configured to *not* fsync() on all Append() calls
16:30:04.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:04.022382 3302 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:30:04.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:04.023842 3302 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: Bootstrap complete.
16:30:04.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:04.054412 3302 raft_consensus.cc:359] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:30:04.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:04.057138 3302 raft_consensus.cc:740] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 25d8bf1b13a0406ba46cf4eebc045b13, State: Initialized, Role: FOLLOWER
16:30:04.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:04.058216 3302 consensus_queue.cc:260] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:30:04.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:04.061771 3306 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:30:04.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:04.063177 3306 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: This master's current role is: FOLLOWER
16:30:04.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:04.066641 3302 sys_catalog.cc:565] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: configured and running, proceeding with master startup.
16:30:04.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:04.126988 3317 catalog_manager.cc:1269] Loaded cluster ID: d116fbc0cf314826bec437e0b8601c6f
16:30:04.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:04.127307 3317 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: loading cluster ID for follower catalog manager: success
16:30:04.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:04.135746 3317 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: acquiring CA information for follower catalog manager: success
16:30:04.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:30:04 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773765003, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.190@KRBTEST.COM for kudu/127.2.129.190@KRBTEST.COM
16:30:04.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:04.143580 3317 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:30:04.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:04.168710 3229 master_runner.cc:186] Error getting master registration for 127.2.129.189:44571: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111),
16:30:04.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:04.195060 3229 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:30:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:04.442317 3304 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:30:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:04.443105 3304 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:30:04.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:04.443997 3304 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:30:04.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:04.483543 3304 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:30:04.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:04.483902 3304 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:30:04.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:04.484380 3304 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:30:04.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:04.485047 3304 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:30:04.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:04.485481 3304 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:30:04.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:04.486075 3304 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:30:04.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:04.486368 3304 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33827/jwks.json
16:30:04.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:04.536201 3304 master_runner.cc:386] Master server non-default flags:
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.129.148:46059
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/wal
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.129.190:46287,127.2.129.189:44571,127.2.129.188:45547
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.129.188
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.129.188:45547
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33827/jwks.json
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data/info.pb
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.129.188
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44951
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/logs
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:30:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:30:04.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:30:04.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:30:04.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:30:04.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:30:04.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:04.538591 3304 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:30:04.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:04.540194 3304 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:30:04.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:04.559616 3326 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:05.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:05.683725 3333 raft_consensus.cc:493] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:30:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:05.684367 3333 raft_consensus.cc:515] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:30:05.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:05.706642 3251 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111)
16:30:05.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:05.711248 3333 leader_election.cc:290] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers b9d3a7751ec842c682fce82d7e5f73f6 (127.2.129.189:44571), 84bd0f13a4024b3bac729dee187bce29 (127.2.129.188:45547)
16:30:05.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:05.723384 3251 leader_election.cc:336] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer b9d3a7751ec842c682fce82d7e5f73f6 (127.2.129.189:44571): Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111)
16:30:05.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:05.726353 3253 leader_election.cc:336] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 84bd0f13a4024b3bac729dee187bce29 (127.2.129.188:45547): Network error: Client connection negotiation failed: client connection to 127.2.129.188:45547: connect: Connection refused (error 111)
16:30:05.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:05.727056 3253 leader_election.cc:304] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 25d8bf1b13a0406ba46cf4eebc045b13; no voters: 84bd0f13a4024b3bac729dee187bce29, b9d3a7751ec842c682fce82d7e5f73f6
16:30:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:05.728107 3333 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
16:30:07.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:05.960690 3325 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 3304
16:30:07.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:07.019333 3325 kernel_stack_watchdog.cc:198] Thread 3304 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 393ms:
16:30:07.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:30:07.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:30:07.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:30:07.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:30:07.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:07.021807 3329 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:07.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:04.566324 3327 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:07.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:07.021184 3304 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.456s user 0.800s sys 1.415s
16:30:07.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:07.023334 3304 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.459s user 0.801s sys 1.416s
16:30:07.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:07.036245 3328 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2469 milliseconds
16:30:07.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.036306 3304 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:30:07.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.037842 3304 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:30:07.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.045950 3304 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:30:07.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.047546 3304 hybrid_clock.cc:648] HybridClock initialized: now 1773765007047470 us; error 86 us; skew 500 ppm
16:30:07.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.048566 3304 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:30:07.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:30:07 dist-test-slave-jdhl krb5kdc[2577](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773765007, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:30:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.056694 3304 init.cc:377] Logged in from keytab as kudu/127.2.129.188@KRBTEST.COM (short username kudu)
16:30:07.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.063159 3304 webserver.cc:492] Webserver started at http://127.2.129.188:44951/ using document root <none> and password file <none>
16:30:07.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.064256 3304 fs_manager.cc:362] Metadata directory not provided
16:30:07.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.064512 3304 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:30:07.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.074069 3304 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.001s
16:30:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.080603 3343 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:30:07.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.082857 3304 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.006s sys 0.000s
16:30:07.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.083343 3304 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/wal
16:30:07.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84bd0f13a4024b3bac729dee187bce29"
16:30:07.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:47 on dist-test-slave-jdhl"
16:30:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.086706 3304 fs_report.cc:389] FS layout report
16:30:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:30:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/wal
16:30:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/wal
16:30:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data/data
16:30:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:30:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:30:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:30:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:30:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:30:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:30:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:30:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:30:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:30:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:30:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:30:07.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.163077 3304 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:30:07.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.174521 3304 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:30:07.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.175222 3304 kserver.cc:163] Server-wide thread pool size limit: 3276
16:30:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.289335 3304 rpc_server.cc:307] RPC server started. Bound to: 127.2.129.188:45547
16:30:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.289534 3395 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.129.188:45547 every 8 connection(s)
16:30:07.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.292526 3304 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-2/data/info.pb
16:30:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.299537 2566 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 3304
16:30:07.300 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:30:07.300 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.129.189:44571
16:30:07.300 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
16:30:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.302347 2566 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/wal
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/logs
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data/info.pb
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.129.189:44571
16:30:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.129.189
16:30:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36995
16:30:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab
16:30:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.129.189
16:30:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:30:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:30:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:30:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:30:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.129.148:46059
16:30:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.129.190:46287,127.2.129.189:44571,127.2.129.188:45547
16:30:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33827/jwks.json
16:30:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:30:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:30:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.310205 3396 sys_catalog.cc:263] Verifying existing consensus state
16:30:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.320547 3396 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: Bootstrap starting.
16:30:07.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.378091 3396 log.cc:826] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: Log is configured to *not* fsync() on all Append() calls
16:30:07.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.413506 3396 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:30:07.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.414947 3396 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: Bootstrap complete.
16:30:07.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.424456 3400 raft_consensus.cc:493] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:30:07.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.425238 3400 raft_consensus.cc:515] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:30:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.442910 3400 leader_election.cc:290] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers b9d3a7751ec842c682fce82d7e5f73f6 (127.2.129.189:44571), 84bd0f13a4024b3bac729dee187bce29 (127.2.129.188:45547)
16:30:07.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:07.457887 3251 leader_election.cc:336] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer b9d3a7751ec842c682fce82d7e5f73f6 (127.2.129.189:44571): Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111)
16:30:07.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.461930 3396 raft_consensus.cc:359] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:30:07.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.465461 3396 raft_consensus.cc:740] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84bd0f13a4024b3bac729dee187bce29, State: Initialized, Role: FOLLOWER
16:30:07.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.466806 3396 consensus_queue.cc:260] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:30:07.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:30:07 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773765003, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.190@KRBTEST.COM for kudu/127.2.129.188@KRBTEST.COM
16:30:07.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.475642 3405 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:30:07.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.476504 3405 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: This master's current role is: FOLLOWER
16:30:07.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.480444 3396 sys_catalog.cc:565] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: configured and running, proceeding with master startup.
16:30:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.497359 3370 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "84bd0f13a4024b3bac729dee187bce29" is_pre_election: true
16:30:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.498478 3370 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 25d8bf1b13a0406ba46cf4eebc045b13 in term 2.
16:30:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.500541 3253 leader_election.cc:304] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 25d8bf1b13a0406ba46cf4eebc045b13, 84bd0f13a4024b3bac729dee187bce29; no voters: b9d3a7751ec842c682fce82d7e5f73f6
16:30:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.501510 3400 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 2 FOLLOWER]: Leader pre-election won for term 3
16:30:07.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.502036 3400 raft_consensus.cc:493] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:30:07.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.502492 3400 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 2 FOLLOWER]: Advancing to term 3
16:30:07.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.526932 3400 raft_consensus.cc:515] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:30:07.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.532073 3400 leader_election.cc:290] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 3 election: Requested vote from peers b9d3a7751ec842c682fce82d7e5f73f6 (127.2.129.189:44571), 84bd0f13a4024b3bac729dee187bce29 (127.2.129.188:45547)
16:30:07.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.532989 3370 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "84bd0f13a4024b3bac729dee187bce29"
16:30:07.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.533605 3370 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 2 FOLLOWER]: Advancing to term 3
16:30:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:07.539551 3251 leader_election.cc:336] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer b9d3a7751ec842c682fce82d7e5f73f6 (127.2.129.189:44571): Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111)
16:30:07.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.557844 3370 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 25d8bf1b13a0406ba46cf4eebc045b13 in term 3.
16:30:07.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.559229 3253 leader_election.cc:304] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 25d8bf1b13a0406ba46cf4eebc045b13, 84bd0f13a4024b3bac729dee187bce29; no voters: b9d3a7751ec842c682fce82d7e5f73f6
16:30:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.560748 3400 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 3 FOLLOWER]: Leader election won for term 3
16:30:07.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.577291 3400 raft_consensus.cc:697] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [term 3 LEADER]: Becoming Leader. State: Replica: 25d8bf1b13a0406ba46cf4eebc045b13, State: Running, Role: LEADER
16:30:07.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.578948 3400 consensus_queue.cc:237] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } }
16:30:07.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.585165 3417 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
16:30:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.615718 3414 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 25d8bf1b13a0406ba46cf4eebc045b13. Latest consensus state: current_term: 3 leader_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:30:07.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.618376 3414 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: This master's current role is: LEADER
16:30:07.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:30:07 dist-test-slave-jdhl krb5kdc[2577](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773765007, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.188@KRBTEST.COM for kudu/127.2.129.190@KRBTEST.COM
16:30:07.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.646133 3423 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
16:30:07.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.659001 3423 catalog_manager.cc:1494] Initializing Kudu cluster ID...
16:30:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.664949 3423 catalog_manager.cc:1269] Loaded cluster ID: d116fbc0cf314826bec437e0b8601c6f
16:30:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.665642 3423 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
16:30:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.671737 3423 catalog_manager.cc:1514] Loading token signing keys...
16:30:07.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.676950 3423 catalog_manager.cc:6055] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13: Loaded TSK: 0
16:30:07.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.678952 3423 catalog_manager.cc:1524] Initializing in-progress tserver states...
16:30:07.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.669163 3304 master_runner.cc:186] Error getting master registration for 127.2.129.189:44571: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111),
16:30:07.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.694864 3304 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:30:07.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:07.899861 3398 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:30:07.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:07.900796 3398 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:30:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:07.902128 3398 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:30:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:07.951984 3398 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:30:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:07.952284 3398 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:30:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:07.952612 3398 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:30:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:07.952929 3398 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:30:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:07.953210 3398 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:30:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:07.953434 3398 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:30:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:07.953598 3398 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33827/jwks.json
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.996748 3398 master_runner.cc:386] Master server non-default flags:
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.129.148:46059
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/wal
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.129.190:46287,127.2.129.189:44571,127.2.129.188:45547
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/kudu.keytab
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.129.189
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.129.189:44571
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33827/jwks.json
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data/info.pb
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.129.189
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36995
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/logs
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:30:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:30:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:30:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:30:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
16:30:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:30:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Mar 2026 16:02:59 UTC on bdcb31816ec0
16:30:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10975
16:30:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:30:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.998108 3398 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:30:07.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:07.999238 3398 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:30:08.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:08.013326 3432 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:08.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:08.095933 3370 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [term 3 FOLLOWER]: Refusing update from remote peer 25d8bf1b13a0406ba46cf4eebc045b13: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
16:30:08.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:08.099627 3414 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.001s
16:30:08.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:08.133618 3438 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 25d8bf1b13a0406ba46cf4eebc045b13. Latest consensus state: current_term: 3 leader_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:30:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:08.139972 3251 consensus_peers.cc:597] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 -> Peer b9d3a7751ec842c682fce82d7e5f73f6 (127.2.129.189:44571): Couldn't send request to peer b9d3a7751ec842c682fce82d7e5f73f6. Status: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
16:30:08.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:08.143949 3438 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: This master's current role is: FOLLOWER
16:30:08.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:08.158933 3438 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:30:08.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:08.160451 3438 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29 [sys.catalog]: This master's current role is: FOLLOWER
16:30:08.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:08.160312 3414 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "25d8bf1b13a0406ba46cf4eebc045b13" member_type: VOTER last_known_addr { host: "127.2.129.190" port: 46287 } } peers { permanent_uuid: "b9d3a7751ec842c682fce82d7e5f73f6" member_type: VOTER last_known_addr { host: "127.2.129.189" port: 44571 } } peers { permanent_uuid: "84bd0f13a4024b3bac729dee187bce29" member_type: VOTER last_known_addr { host: "127.2.129.188" port: 45547 } } }
16:30:08.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:08.161311 3414 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25d8bf1b13a0406ba46cf4eebc045b13 [sys.catalog]: This master's current role is: LEADER
16:30:08.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:08.599095 3417 catalog_manager.cc:1269] Loaded cluster ID: d116fbc0cf314826bec437e0b8601c6f
16:30:08.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:08.599560 3417 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: loading cluster ID for follower catalog manager: success
16:30:08.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:08.611508 3417 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: acquiring CA information for follower catalog manager: success
16:30:08.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:08.622633 3417 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 84bd0f13a4024b3bac729dee187bce29: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:30:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.202119 3229 master_runner.cc:186] Error getting master registration for 127.2.129.189:44571: Network error: Client connection negotiation failed: client connection to 127.2.129.189:44571: connect: Connection refused (error 111),
16:30:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.202618 3229 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:30:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:09.417050 3430 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 3398
16:30:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:08.015641 3431 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:09.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:09.748827 3398 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.736s user 0.570s sys 0.932s
16:30:09.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:09.749301 3398 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.738s user 0.570s sys 0.932s
16:30:09.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.749604 3398 server_base.cc:1060] running on GCE node
16:30:09.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:09.759438 3434 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.761299 3398 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:30:09.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.766216 3398 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:30:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.769158 3398 hybrid_clock.cc:648] HybridClock initialized: now 1773765009769062 us; error 81 us; skew 500 ppm
16:30:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.770869 3398 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:30:09.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 17 16:30:09 dist-test-slave-jdhl krb5kdc[2577](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773765009, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.129.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:30:09.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.779471 3398 init.cc:377] Logged in from keytab as kudu/127.2.129.189@KRBTEST.COM (short username kudu)
16:30:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.789068 3398 webserver.cc:492] Webserver started at http://127.2.129.189:36995/ using document root <none> and password file <none>
16:30:09.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.790762 3398 fs_manager.cc:362] Metadata directory not provided
16:30:09.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.791154 3398 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:30:09.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.802886 3398 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.001s
16:30:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.813905 3452 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:30:09.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.815593 3398 fs_manager.cc:730] Time spent opening block manager: real 0.010s user 0.007s sys 0.001s
16:30:09.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.815975 3398 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data,/tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/wal
16:30:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9d3a7751ec842c682fce82d7e5f73f6"
16:30:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-17 16:29:45 on dist-test-slave-jdhl"
16:30:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.818498 3398 fs_report.cc:389] FS layout report
16:30:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:30:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/wal
16:30:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/wal
16:30:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data/data
16:30:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:30:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:30:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:30:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:30:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:30:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:30:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:30:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:30:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:30:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:30:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:30:09.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.958000 3398 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:30:09.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.966378 3398 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:30:09.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:09.966965 3398 kserver.cc:163] Server-wide thread pool size limit: 3276
16:30:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:10.117576 3398 rpc_server.cc:307] RPC server started. Bound to: 127.2.129.189:44571
16:30:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:10.117657 3508 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.129.189:44571 every 8 connection(s)
16:30:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:10.121191 3398 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/master-1/data/info.pb
16:30:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:10.128364 2566 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu as pid 3398
16:30:10.130 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:30:10.131 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.129.190:46287(127.2.129.190:46287)] connecting to peer
16:30:10.140 [DEBUG - kudu-nio-3] (Connection.java:757) [peer master-127.2.129.190:46287(127.2.129.190:46287)] Successfully connected to peer
16:30:10.156 [DEBUG - kudu-nio-3] (SecurityContext.java:222) Refreshing Kerberos credentials...
16:30:10.157 [DEBUG - kudu-nio-3] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskMUvRhB/test-tmp/mini-kudu-cluster9505359300785283590/krb5kdc/krb5cc
16:30:10.159 [DEBUG - kudu-nio-3] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
16:30:10.159 [WARN - kudu-nio-3] (SecurityContext.java:231) Tried to refresh Kerberos credentials but was unable to re-login from ticket cache
16:30:10.160 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.2.129.190:46287(127.2.129.190:46287)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
16:30:10.161 [DEBUG - kudu-nio-3] (Connection.java:239) [peer master-127.2.129.190:46287(127.2.129.190:46287)] handling channelInactive
16:30:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:10.161440 3509 sys_catalog.cc:263] Verifying existing consensus state
16:30:10.162 [DEBUG - main] (Connection.java:575) [peer master-127.2.129.188:45547(127.2.129.188:45547)] disconnecting while in state TERMINATED
16:30:10.162 [DEBUG - main] (Connection.java:575) [peer master-127.2.129.189:44571(127.2.129.189:44571)] disconnecting while in state TERMINATED
16:30:10.162 [DEBUG - main] (Connection.java:575) [peer master-127.2.129.190:46287(127.2.129.190:46287)] disconnecting while in state TERMINATED
16:30:10.162 [DEBUG - kudu-nio-3] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:30:10.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:10.166390 3512 negotiation.cc:337] Failed RPC negotiation. Trace:
16:30:10.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:30:10.139911 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:42222 (local address 127.2.129.190:46287)
16:30:10.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:30:10.154487 (+ 14576us) server_negotiation.cc:207] Beginning negotiation
16:30:10.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:30:10.154518 (+ 31us) server_negotiation.cc:400] Waiting for connection header
16:30:10.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:30:10.154578 (+ 60us) server_negotiation.cc:408] Connection header received
16:30:10.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:30:10.154736 (+ 158us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
16:30:10.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:30:10.154765 (+ 29us) server_negotiation.cc:462] Received NEGOTIATE request from client
16:30:10.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:30:10.155039 (+ 274us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
16:30:10.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:30:10.161948 (+ 6909us) server_negotiation.cc:227] Negotiated authn=SASL
16:30:10.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0317 16:30:10.162733 (+ 785us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:42222: BlockingRecv error: recv got EOF from 127.0.0.1:42222 (error 108)
16:30:10.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":13904,"thread_start_us":13557,"threads_started":1}
16:30:10.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:10.167207 2566 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 3229
16:30:10.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:10.177397 3509 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b9d3a7751ec842c682fce82d7e5f73f6: Bootstrap starting.
16:30:10.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:10.233865 2566 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 3398
16:30:10.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:10.261922 2566 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 3304
16:30:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-17T16:30:11Z chronyd exiting
16:30:12.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:12.079800 2410 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad6c0 after lost signal to thread 2284
16:30:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:12.080721 2410 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9040 after lost signal to thread 2413
16:30:12.808 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:30:12.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:12.810927 1836 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 2130
16:30:12.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:12.853389 1836 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 2283
16:30:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:12.897650 1836 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 2424
16:30:12.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:12.928746 1836 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 1848
16:30:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:12.963467 1836 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 1919
16:30:12.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:12.973366 2023 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.203.62:41027: connect: Connection refused (error 111)
16:30:12.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260317 16:30:12.976967 2023 consensus_peers.cc:597] T 00000000000000000000000000000000 P 960b40c8905e461bad3454144ce157f4 -> Peer c6539747b8fe4235b67200c0648fe78e (127.1.203.62:41027): Couldn't send request to peer c6539747b8fe4235b67200c0648fe78e. Status: Network error: Client connection negotiation failed: client connection to 127.1.203.62:41027: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
16:30:13.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260317 16:30:13.000679 1836 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskMUvRhB/build/tsan/bin/kudu with pid 1997
16:30:14.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-17T16:30:14Z chronyd exiting
16:30:15.302 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more