14:10:38.916 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:10:38.928 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskLlQHbR/test-tmp
14:10:38.933 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:10:38.937 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskLlQHbR/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:10:39.239 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:10:39.421 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:10:39.422 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:10:39.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:39.426308 1808 mini_oidc.cc:150] Starting JWKS server
14:10:39.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:39.430588 1808 webserver.cc:492] Webserver started at http://127.0.0.1:42941/ using document root <none> and password file <none>
14:10:39.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:39.431165 1808 mini_oidc.cc:186] Starting OIDC Discovery server
14:10:39.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:39.433157 1808 webserver.cc:492] Webserver started at http://127.0.0.1:40209/ using document root <none> and password file <none>
14:10:39.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-04T14:10:39Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:10:39.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-04T14:10:39Z Disabled control of system clock
14:10:39.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:39.496503 1808 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:10:39.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:10:39.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-0/wal
14:10:39.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-0/data
14:10:39.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:39.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:39.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:39.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:39.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:39.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:39.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-0/logs
14:10:39.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-0/data/info.pb
14:10:39.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:39.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:39.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:39.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:39.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:39.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:39.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:39.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:39.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:39.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:39.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:39.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:39.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.196.62:34245
14:10:39.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.196.62
14:10:39.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:39.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.196.20:37005
14:10:39.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:39.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:39.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:39.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:39.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.196.62:34245,127.1.196.61:40113,127.1.196.60:38475
14:10:39.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:39.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42941/jwks.json with env {}
14:10:39.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:39.819136 1820 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:10:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:39.819752 1820 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:39.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:39.820421 1820 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:10:39.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:39.854889 1820 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:10:39.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:39.855331 1820 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:10:39.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:39.855602 1820 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:39.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:39.855943 1820 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:10:39.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:39.856277 1820 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:39.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:39.856570 1820 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42941/jwks.json
14:10:39.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:39.896495 1820 master_runner.cc:386] Master server non-default flags:
14:10:39.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:39.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.196.20:37005
14:10:39.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:39.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:39.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-0/data
14:10:39.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-0/wal
14:10:39.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:39.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.196.62:34245,127.1.196.61:40113,127.1.196.60:38475
14:10:39.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:39.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:39.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:39.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.196.62:34245
14:10:39.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:39.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:39.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:39.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42941/jwks.json
14:10:39.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:39.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:39.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-0/data/info.pb
14:10:39.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.196.62
14:10:39.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:39.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:39.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:39.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:39.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:39.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:39.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-0/logs
14:10:39.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:39.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:39.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:39.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:39.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:10:39.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:10:39.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:39.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:10:39.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:10:39.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:39.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:39.898622 1820 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:39.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:39.900503 1820 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:39.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:39.917472 1826 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:41.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:41.320436 1825 debug-util.cc:398] Leaking SignalData structure 0x7b0800034cc0 after lost signal to thread 1820
14:10:41.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:39.919587 1827 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:41.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:41.766830 1825 kernel_stack_watchdog.cc:198] Thread 1820 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
14:10:41.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:10:41.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:10:41.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:10:41.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:10:41.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:41.767567 1820 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.848s user 0.671s sys 1.175s
14:10:41.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:41.768286 1820 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.849s user 0.672s sys 1.175s
14:10:41.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:41.770084 1829 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:41.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:41.777745 1828 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1848 milliseconds
14:10:41.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.777894 1820 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:41.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.779423 1820 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:41.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.782174 1820 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:41.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.783595 1820 hybrid_clock.cc:648] HybridClock initialized: now 1777903841783562 us; error 42 us; skew 500 ppm
14:10:41.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.784569 1820 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:41.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.791435 1820 webserver.cc:492] Webserver started at http://127.1.196.62:39549/ using document root <none> and password file <none>
14:10:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.792744 1820 fs_manager.cc:362] Metadata directory not provided
14:10:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.793021 1820 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.793651 1820 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:10:41.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.798243 1820 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-0/data/instance:
14:10:41.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33fdaeff36e742b0a90625272aa165f0"
14:10:41.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:41 on dist-test-slave-vb5z"
14:10:41.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.799746 1820 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-0/wal/instance:
14:10:41.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33fdaeff36e742b0a90625272aa165f0"
14:10:41.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:41 on dist-test-slave-vb5z"
14:10:41.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.808214 1820 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.002s
14:10:41.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.815207 1836 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:41.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.816543 1820 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
14:10:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.816952 1820 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-0/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-0/wal
14:10:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33fdaeff36e742b0a90625272aa165f0"
14:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:41 on dist-test-slave-vb5z"
14:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.817423 1820 fs_report.cc:389] FS layout report
14:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-0/wal
14:10:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-0/wal
14:10:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-0/data/data
14:10:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:41.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:41.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:41.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:41.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:41.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:41.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:41.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:41.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:41.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:41.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.875599 1820 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:41.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.877436 1820 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:41.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.878144 1820 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:41.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.965540 1820 rpc_server.cc:307] RPC server started. Bound to: 127.1.196.62:34245
14:10:41.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.965616 1887 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.196.62:34245 every 8 connection(s)
14:10:41.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.969023 1820 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-0/data/info.pb
14:10:41.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.972513 1808 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 1820
14:10:41.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.973167 1808 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-0/wal/instance
14:10:41.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.976182 1888 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:10:41.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.976706 1808 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:10:41.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:10:41.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-1/wal
14:10:41.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-1/data
14:10:41.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:41.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:41.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:41.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:41.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:41.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:41.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-1/logs
14:10:41.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-1/data/info.pb
14:10:41.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:41.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:41.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:41.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:41.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:41.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:41.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:41.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:41.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:41.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:41.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.196.61:40113
14:10:41.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.196.61
14:10:41.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:41.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.196.20:37005
14:10:41.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:41.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:41.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:41.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:41.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.196.62:34245,127.1.196.61:40113,127.1.196.60:38475
14:10:41.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:41.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42941/jwks.json with env {}
14:10:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:41.992606 1888 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } has no permanent_uuid. Determining permanent_uuid...
14:10:42.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:42.024518 1888 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } has no permanent_uuid. Determining permanent_uuid...
14:10:42.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.027405 1838 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.196.61:40113: connect: Connection refused (error 111)
14:10:42.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.032871 1888 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.61:40113: Network error: Client connection negotiation failed: client connection to 127.1.196.61:40113: connect: Connection refused (error 111)
14:10:42.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:42.082401 1888 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } attempt: 1
14:10:42.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.087175 1888 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.61:40113: Network error: Client connection negotiation failed: client connection to 127.1.196.61:40113: connect: Connection refused (error 111)
14:10:42.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:42.155735 1888 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } attempt: 2
14:10:42.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.161182 1888 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.61:40113: Network error: Client connection negotiation failed: client connection to 127.1.196.61:40113: connect: Connection refused (error 111)
14:10:42.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:42.252741 1888 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } attempt: 3
14:10:42.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.257120 1888 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.61:40113: Network error: Client connection negotiation failed: client connection to 127.1.196.61:40113: connect: Connection refused (error 111)
14:10:42.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.331259 1890 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:10:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.332033 1890 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.332563 1890 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:10:42.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.368729 1890 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:10:42.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.369027 1890 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:10:42.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.369285 1890 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:42.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.369483 1890 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:10:42.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.369786 1890 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:42.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.370074 1890 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42941/jwks.json
14:10:42.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:42.400525 1888 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } attempt: 4
14:10:42.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.404626 1888 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.61:40113: Network error: Client connection negotiation failed: client connection to 127.1.196.61:40113: connect: Connection refused (error 111)
14:10:42.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:42.408412 1890 master_runner.cc:386] Master server non-default flags:
14:10:42.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:42.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.196.20:37005
14:10:42.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:42.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:42.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-1/data
14:10:42.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-1/wal
14:10:42.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:42.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.196.62:34245,127.1.196.61:40113,127.1.196.60:38475
14:10:42.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:42.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:42.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:42.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.196.61:40113
14:10:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42941/jwks.json
14:10:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:42.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:42.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-1/data/info.pb
14:10:42.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.196.61
14:10:42.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:42.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:42.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-1/logs
14:10:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:10:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:10:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:10:42.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:10:42.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:42.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:42.409803 1890 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:42.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:42.411569 1890 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:42.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.425457 1899 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:42.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:42.704156 1888 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } attempt: 5
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.713562 1888 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.61:40113: Network error: Client connection negotiation failed: client connection to 127.1.196.61:40113: connect: Connection refused (error 111)
14:10:43.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.261209 1888 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } attempt: 6
14:10:43.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:43.271600 1888 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.61:40113: Network error: Client connection negotiation failed: client connection to 127.1.196.61:40113: connect: Connection refused (error 111)
14:10:43.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.430097 1902 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:43.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:42.426939 1900 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:43.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:43.721235 1901 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1289 milliseconds
14:10:43.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.721359 1890 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:43.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.722558 1890 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.725339 1890 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:43.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.726720 1890 hybrid_clock.cc:648] HybridClock initialized: now 1777903843726674 us; error 81 us; skew 500 ppm
14:10:43.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.727561 1890 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:43.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.734211 1890 webserver.cc:492] Webserver started at http://127.1.196.61:45465/ using document root <none> and password file <none>
14:10:43.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.735252 1890 fs_manager.cc:362] Metadata directory not provided
14:10:43.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.735515 1890 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:43.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.735935 1890 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:10:43.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.740398 1890 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-1/data/instance:
14:10:43.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "887fd885bc804fb9b5e87fc63a706826"
14:10:43.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:43 on dist-test-slave-vb5z"
14:10:43.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.741562 1890 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-1/wal/instance:
14:10:43.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "887fd885bc804fb9b5e87fc63a706826"
14:10:43.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:43 on dist-test-slave-vb5z"
14:10:43.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.750051 1890 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
14:10:43.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.756983 1911 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:43.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.758296 1890 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
14:10:43.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.758618 1890 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-1/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-1/wal
14:10:43.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "887fd885bc804fb9b5e87fc63a706826"
14:10:43.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:43 on dist-test-slave-vb5z"
14:10:43.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.758950 1890 fs_report.cc:389] FS layout report
14:10:43.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:43.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-1/wal
14:10:43.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-1/wal
14:10:43.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-1/data/data
14:10:43.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:43.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:43.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:43.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:43.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:43.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:43.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:43.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:43.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:43.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:43.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:43.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.827224 1890 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:43.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.829355 1890 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:43.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.829793 1890 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:43.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.917604 1890 rpc_server.cc:307] RPC server started. Bound to: 127.1.196.61:40113
14:10:43.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.917678 1962 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.196.61:40113 every 8 connection(s)
14:10:43.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.920394 1890 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-1/data/info.pb
14:10:43.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.927546 1963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:10:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.929251 1808 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 1890
14:10:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.929569 1808 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-1/wal/instance
14:10:43.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.931773 1808 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:10:43.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:10:43.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-2/wal
14:10:43.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-2/data
14:10:43.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:43.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:43.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:43.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:43.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:43.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:43.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-2/logs
14:10:43.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-2/data/info.pb
14:10:43.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:43.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:43.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:43.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:43.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:43.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:43.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:43.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:43.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:43.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:43.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:43.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:43.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.196.60:38475
14:10:43.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.196.60
14:10:43.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:43.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.196.20:37005
14:10:43.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:43.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:43.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:43.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:43.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.196.62:34245,127.1.196.61:40113,127.1.196.60:38475
14:10:43.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:43.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42941/jwks.json with env {}
14:10:43.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.941136 1963 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } has no permanent_uuid. Determining permanent_uuid...
14:10:43.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.966331 1963 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } has no permanent_uuid. Determining permanent_uuid...
14:10:43.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:43.990329 1963 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } has no permanent_uuid. Determining permanent_uuid...
14:10:43.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:43.993130 1915 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.196.60:38475: connect: Connection refused (error 111)
14:10:43.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:43.998556 1963 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.60:38475: Network error: Client connection negotiation failed: client connection to 127.1.196.60:38475: connect: Connection refused (error 111)
14:10:44.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:44.048198 1963 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } attempt: 1
14:10:44.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.053507 1963 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.60:38475: Network error: Client connection negotiation failed: client connection to 127.1.196.60:38475: connect: Connection refused (error 111)
14:10:44.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:44.122018 1963 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } attempt: 2
14:10:44.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.127928 1963 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.60:38475: Network error: Client connection negotiation failed: client connection to 127.1.196.60:38475: connect: Connection refused (error 111)
14:10:44.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:44.219488 1963 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } attempt: 3
14:10:44.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.224020 1963 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.60:38475: Network error: Client connection negotiation failed: client connection to 127.1.196.60:38475: connect: Connection refused (error 111)
14:10:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.274664 1965 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:10:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.275244 1965 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:44.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.275805 1965 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:10:44.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.309422 1965 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:10:44.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.309732 1965 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:10:44.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.309931 1965 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:44.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.310115 1965 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:10:44.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.310293 1965 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:44.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.310473 1965 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42941/jwks.json
14:10:44.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:44.332201 1888 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } attempt: 7
14:10:44.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:44.344669 1888 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } has no permanent_uuid. Determining permanent_uuid...
14:10:44.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.350633 1888 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.60:38475: Network error: Client connection negotiation failed: client connection to 127.1.196.60:38475: connect: Connection refused (error 111)
14:10:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:44.353233 1965 master_runner.cc:386] Master server non-default flags:
14:10:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:44.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.196.20:37005
14:10:44.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:44.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:44.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-2/data
14:10:44.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-2/wal
14:10:44.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:44.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.196.62:34245,127.1.196.61:40113,127.1.196.60:38475
14:10:44.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:44.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:44.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:44.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.196.60:38475
14:10:44.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:44.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:44.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:44.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42941/jwks.json
14:10:44.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:44.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:44.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-2/data/info.pb
14:10:44.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.196.60
14:10:44.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:44.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:44.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:44.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:44.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:44.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:44.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-2/logs
14:10:44.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:44.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:44.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:44.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:44.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:10:44.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:10:44.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:44.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:10:44.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:10:44.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:44.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:44.354877 1965 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:44.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:44.357129 1965 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:44.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:44.367451 1963 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } attempt: 4
14:10:44.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.369752 1977 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:44.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.373764 1963 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.60:38475: Network error: Client connection negotiation failed: client connection to 127.1.196.60:38475: connect: Connection refused (error 111)
14:10:44.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:44.409075 1888 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } attempt: 1
14:10:44.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.415809 1888 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.60:38475: Network error: Client connection negotiation failed: client connection to 127.1.196.60:38475: connect: Connection refused (error 111)
14:10:44.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:44.497395 1888 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } attempt: 2
14:10:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.504892 1888 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.60:38475: Network error: Client connection negotiation failed: client connection to 127.1.196.60:38475: connect: Connection refused (error 111)
14:10:44.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:44.590500 1888 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } attempt: 3
14:10:44.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.598122 1888 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.60:38475: Network error: Client connection negotiation failed: client connection to 127.1.196.60:38475: connect: Connection refused (error 111)
14:10:44.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:44.673216 1963 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } attempt: 5
14:10:44.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.680263 1963 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.60:38475: Network error: Client connection negotiation failed: client connection to 127.1.196.60:38475: connect: Connection refused (error 111)
14:10:44.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:44.738727 1888 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } attempt: 4
14:10:44.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.745704 1888 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.60:38475: Network error: Client connection negotiation failed: client connection to 127.1.196.60:38475: connect: Connection refused (error 111)
14:10:45.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.029372 1888 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } attempt: 5
14:10:45.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:45.036211 1888 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.60:38475: Network error: Client connection negotiation failed: client connection to 127.1.196.60:38475: connect: Connection refused (error 111)
14:10:45.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.227947 1963 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } attempt: 6
14:10:45.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:45.239600 1963 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.60:38475: Network error: Client connection negotiation failed: client connection to 127.1.196.60:38475: connect: Connection refused (error 111)
14:10:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.588801 1888 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } attempt: 6
14:10:45.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:45.598970 1888 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.196.60:38475: Network error: Client connection negotiation failed: client connection to 127.1.196.60:38475: connect: Connection refused (error 111)
14:10:45.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:44.371496 1978 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:45.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:45.653069 1980 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:45.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:45.655719 1965 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.282s user 0.001s sys 0.012s
14:10:45.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:45.656040 1965 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.283s user 0.001s sys 0.012s
14:10:45.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.656291 1965 server_base.cc:1061] running on GCE node
14:10:45.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.657574 1965 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:45.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.661010 1965 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:45.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.662469 1965 hybrid_clock.cc:648] HybridClock initialized: now 1777903845662405 us; error 78 us; skew 500 ppm
14:10:45.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.665634 1965 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:45.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.675880 1965 webserver.cc:492] Webserver started at http://127.1.196.60:41469/ using document root <none> and password file <none>
14:10:45.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.677381 1965 fs_manager.cc:362] Metadata directory not provided
14:10:45.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.677731 1965 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:45.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.678745 1965 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:10:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.683799 1965 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-2/data/instance:
14:10:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f5f595840efb4a9ea041a0dee1414a19"
14:10:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:45 on dist-test-slave-vb5z"
14:10:45.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.685003 1965 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-2/wal/instance:
14:10:45.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f5f595840efb4a9ea041a0dee1414a19"
14:10:45.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:45 on dist-test-slave-vb5z"
14:10:45.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.693809 1965 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.004s sys 0.004s
14:10:45.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.701658 1991 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:45.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.702980 1965 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.002s sys 0.003s
14:10:45.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.703302 1965 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-2/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-2/wal
14:10:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f5f595840efb4a9ea041a0dee1414a19"
14:10:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:45 on dist-test-slave-vb5z"
14:10:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.703862 1965 fs_report.cc:389] FS layout report
14:10:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-2/wal
14:10:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-2/wal
14:10:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-2/data/data
14:10:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:45.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:45.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:45.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:45.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:45.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:45.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:45.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:45.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:45.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.784755 1965 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:45.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.786674 1965 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:45.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.787082 1965 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.876556 1965 rpc_server.cc:307] RPC server started. Bound to: 127.1.196.60:38475
14:10:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.876637 2042 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.196.60:38475 every 8 connection(s)
14:10:45.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.879726 1965 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-2/data/info.pb
14:10:45.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.882550 1808 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 1965
14:10:45.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.883184 1808 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/master-2/wal/instance
14:10:45.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.889143 2043 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:10:45.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.911304 2043 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } has no permanent_uuid. Determining permanent_uuid...
14:10:45.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.941267 2043 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } has no permanent_uuid. Determining permanent_uuid...
14:10:45.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.956291 2043 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } has no permanent_uuid. Determining permanent_uuid...
14:10:45.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.976202 2043 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19: Bootstrap starting.
14:10:45.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.986569 2043 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19: Neither blocks nor log segments found. Creating new log.
14:10:45.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.989042 2043 log.cc:826] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19: Log is configured to *not* fsync() on all Append() calls
14:10:45.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:45.994415 2043 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19: No bootstrap required, opened a new log
14:10:46.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.016520 2043 raft_consensus.cc:359] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } }
14:10:46.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.017382 2043 raft_consensus.cc:385] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:10:46.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.017688 2043 raft_consensus.cc:740] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f5f595840efb4a9ea041a0dee1414a19, State: Initialized, Role: FOLLOWER
14:10:46.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.018622 2043 consensus_queue.cc:260] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } }
14:10:46.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.022920 2051 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } } }
14:10:46.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.023705 2051 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [sys.catalog]: This master's current role is: FOLLOWER
14:10:46.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.025051 2043 sys_catalog.cc:565] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [sys.catalog]: configured and running, proceeding with master startup.
14:10:46.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:46.042780 1997 tablet.cc:2404] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:10:46.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:46.059120 2062 catalog_manager.cc:1568] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:10:46.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:46.059533 2062 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:10:46.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.050863 1965 master_runner.cc:186] Error getting master registration for 127.1.196.62:34245: OK, instance_id { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" instance_seqno: 1777903841940425 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:10:46.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.076967 1965 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:46.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.140161 2051 raft_consensus.cc:493] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:10:46.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.140697 2051 raft_consensus.cc:515] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } }
14:10:46.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.143959 2051 leader_election.cc:290] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 33fdaeff36e742b0a90625272aa165f0 (127.1.196.62:34245), 887fd885bc804fb9b5e87fc63a706826 (127.1.196.61:40113)
14:10:46.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.300246 1963 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } attempt: 7
14:10:46.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.325779 1963 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826: Bootstrap starting.
14:10:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.334609 1963 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826: Neither blocks nor log segments found. Creating new log.
14:10:46.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.337955 1963 log.cc:826] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826: Log is configured to *not* fsync() on all Append() calls
14:10:46.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.344113 1963 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826: No bootstrap required, opened a new log
14:10:46.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.373920 1963 raft_consensus.cc:359] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } }
14:10:46.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.375315 1963 raft_consensus.cc:385] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:10:46.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.375644 1963 raft_consensus.cc:740] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 887fd885bc804fb9b5e87fc63a706826, State: Initialized, Role: FOLLOWER
14:10:46.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.376863 1963 consensus_queue.cc:260] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } }
14:10:46.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.379880 2066 sys_catalog.cc:455] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } } }
14:10:46.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.381343 2066 sys_catalog.cc:458] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826 [sys.catalog]: This master's current role is: FOLLOWER
14:10:46.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.382495 1963 sys_catalog.cc:565] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826 [sys.catalog]: configured and running, proceeding with master startup.
14:10:46.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.382704 1938 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f5f595840efb4a9ea041a0dee1414a19" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "887fd885bc804fb9b5e87fc63a706826" is_pre_election: true
14:10:46.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.383637 1938 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f5f595840efb4a9ea041a0dee1414a19 in term 0.
14:10:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.388262 1993 leader_election.cc:304] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 887fd885bc804fb9b5e87fc63a706826, f5f595840efb4a9ea041a0dee1414a19; no voters:
14:10:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.389834 2051 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:10:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.390189 2051 raft_consensus.cc:493] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:10:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.390517 2051 raft_consensus.cc:3060] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [term 0 FOLLOWER]: Advancing to term 1
14:10:46.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.404026 2051 raft_consensus.cc:515] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } }
14:10:46.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.407042 2051 leader_election.cc:290] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [CANDIDATE]: Term 1 election: Requested vote from peers 33fdaeff36e742b0a90625272aa165f0 (127.1.196.62:34245), 887fd885bc804fb9b5e87fc63a706826 (127.1.196.61:40113)
14:10:46.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.407146 1938 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f5f595840efb4a9ea041a0dee1414a19" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "887fd885bc804fb9b5e87fc63a706826"
14:10:46.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.407711 1938 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826 [term 0 FOLLOWER]: Advancing to term 1
14:10:46.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.415326 1938 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f5f595840efb4a9ea041a0dee1414a19 in term 1.
14:10:46.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.416895 1993 leader_election.cc:304] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 887fd885bc804fb9b5e87fc63a706826, f5f595840efb4a9ea041a0dee1414a19; no voters:
14:10:46.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.418138 2051 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [term 1 FOLLOWER]: Leader election won for term 1
14:10:46.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.423625 2051 raft_consensus.cc:697] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [term 1 LEADER]: Becoming Leader. State: Replica: f5f595840efb4a9ea041a0dee1414a19, State: Running, Role: LEADER
14:10:46.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.425130 2051 consensus_queue.cc:237] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } }
14:10:46.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.440438 2074 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f5f595840efb4a9ea041a0dee1414a19. Latest consensus state: current_term: 1 leader_uuid: "f5f595840efb4a9ea041a0dee1414a19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } } }
14:10:46.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.441342 2074 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [sys.catalog]: This master's current role is: LEADER
14:10:46.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.446399 2078 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
14:10:46.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.458125 2078 catalog_manager.cc:1494] Initializing Kudu cluster ID...
14:10:46.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:46.486609 2080 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:10:46.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:46.486994 2080 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:10:46.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.493759 1938 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826 [term 1 FOLLOWER]: Refusing update from remote peer f5f595840efb4a9ea041a0dee1414a19: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:10:46.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.495301 2074 consensus_queue.cc:1048] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [LEADER]: Connected to new peer: Peer: permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
14:10:46.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.455019 1890 master_runner.cc:186] Error getting master registration for 127.1.196.62:34245: OK, instance_id { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" instance_seqno: 1777903841940425 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:10:46.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.515818 1890 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:46.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.530179 2066 sys_catalog.cc:455] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f5f595840efb4a9ea041a0dee1414a19. Latest consensus state: current_term: 1 leader_uuid: "f5f595840efb4a9ea041a0dee1414a19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } } }
14:10:46.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.530951 2066 sys_catalog.cc:458] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826 [sys.catalog]: This master's current role is: FOLLOWER
14:10:46.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.552407 2051 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f5f595840efb4a9ea041a0dee1414a19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } } }
14:10:46.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.553207 2051 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [sys.catalog]: This master's current role is: LEADER
14:10:46.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.555402 2078 catalog_manager.cc:1357] Generated new cluster ID: 0566e41626e94c8490bb6934090e836e
14:10:46.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.555720 2078 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
14:10:46.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.570900 2066 sys_catalog.cc:455] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f5f595840efb4a9ea041a0dee1414a19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } } }
14:10:46.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.571687 2066 sys_catalog.cc:458] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826 [sys.catalog]: This master's current role is: FOLLOWER
14:10:46.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.594587 2078 catalog_manager.cc:1380] Generated new certificate authority record
14:10:46.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.601325 2078 catalog_manager.cc:1514] Loading token signing keys...
14:10:46.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.632969 1888 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } attempt: 7
14:10:46.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.676510 2078 catalog_manager.cc:6044] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19: Generated new TSK 0
14:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.679540 2078 catalog_manager.cc:1524] Initializing in-progress tserver states...
14:10:46.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.694097 1888 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0: Bootstrap starting.
14:10:46.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.703146 1888 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0: Neither blocks nor log segments found. Creating new log.
14:10:46.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.706305 1888 log.cc:826] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0: Log is configured to *not* fsync() on all Append() calls
14:10:46.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.713188 1808 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:10:46.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:10:46.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-0/wal
14:10:46.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-0/data
14:10:46.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:46.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:46.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:46.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:46.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:46.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:46.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-0/logs
14:10:46.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-0/data/info.pb
14:10:46.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:46.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:46.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:46.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:46.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:46.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:46.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:46.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:46.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:10:46.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:46.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.196.1:0
14:10:46.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.196.1
14:10:46.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.196.1
14:10:46.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:46.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.196.62:34245,127.1.196.61:40113,127.1.196.60:38475
14:10:46.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.196.20:37005
14:10:46.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:46.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:46.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:10:46.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.722467 1888 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0: No bootstrap required, opened a new log
14:10:46.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.770488 1888 raft_consensus.cc:359] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } }
14:10:46.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.772321 1888 raft_consensus.cc:385] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:10:46.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.773984 1888 raft_consensus.cc:740] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 33fdaeff36e742b0a90625272aa165f0, State: Initialized, Role: FOLLOWER
14:10:46.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.775971 1888 consensus_queue.cc:260] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } }
14:10:46.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.795969 1888 sys_catalog.cc:565] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0 [sys.catalog]: configured and running, proceeding with master startup.
14:10:46.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.797058 1863 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f5f595840efb4a9ea041a0dee1414a19" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "33fdaeff36e742b0a90625272aa165f0" is_pre_election: true
14:10:46.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.800412 1861 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0 [term 0 FOLLOWER]: Advancing to term 1
14:10:46.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.800460 1862 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f5f595840efb4a9ea041a0dee1414a19" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "33fdaeff36e742b0a90625272aa165f0"
14:10:46.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.791056 2090 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } } }
14:10:46.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.827773 2090 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0 [sys.catalog]: This master's current role is: FOLLOWER
14:10:46.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.824277 1861 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0 [term 1 FOLLOWER]: Refusing update from remote peer f5f595840efb4a9ea041a0dee1414a19: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:10:46.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.839643 2074 consensus_queue.cc:1048] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [LEADER]: Connected to new peer: Peer: permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
14:10:46.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:46.891223 1842 tablet.cc:2404] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:10:46.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.982734 2097 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f5f595840efb4a9ea041a0dee1414a19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } } }
14:10:46.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.984761 2097 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0 [sys.catalog]: This master's current role is: FOLLOWER
14:10:46.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.986827 2090 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f5f595840efb4a9ea041a0dee1414a19. Latest consensus state: current_term: 1 leader_uuid: "f5f595840efb4a9ea041a0dee1414a19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } } }
14:10:46.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:46.992915 2090 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0 [sys.catalog]: This master's current role is: FOLLOWER
14:10:47.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:47.073730 2108 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:10:47.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:47.331765 2074 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f5f595840efb4a9ea041a0dee1414a19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33fdaeff36e742b0a90625272aa165f0" member_type: VOTER last_known_addr { host: "127.1.196.62" port: 34245 } } peers { permanent_uuid: "887fd885bc804fb9b5e87fc63a706826" member_type: VOTER last_known_addr { host: "127.1.196.61" port: 40113 } } peers { permanent_uuid: "f5f595840efb4a9ea041a0dee1414a19" member_type: VOTER last_known_addr { host: "127.1.196.60" port: 38475 } } }
14:10:47.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:47.333644 2074 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5f595840efb4a9ea041a0dee1414a19 [sys.catalog]: This master's current role is: LEADER
14:10:47.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:47.474179 2089 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:10:47.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:47.475332 2089 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:47.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:47.476373 2089 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:10:47.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:47.492584 2080 catalog_manager.cc:1269] Loaded cluster ID: 0566e41626e94c8490bb6934090e836e
14:10:47.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:47.493314 2080 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826: loading cluster ID for follower catalog manager: success
14:10:47.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:47.503504 2080 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826: acquiring CA information for follower catalog manager: success
14:10:47.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:47.520975 2080 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 887fd885bc804fb9b5e87fc63a706826: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:10:47.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:47.559031 2089 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:10:47.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:47.560822 2089 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.196.1
14:10:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:47.630564 2089 tablet_server_runner.cc:78] Tablet server non-default flags:
14:10:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.196.20:37005
14:10:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-0/data
14:10:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-0/wal
14:10:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.196.1:0
14:10:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-0/data/info.pb
14:10:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.196.1
14:10:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.196.62:34245,127.1.196.61:40113,127.1.196.60:38475
14:10:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.196.1
14:10:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-0/logs
14:10:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:10:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:10:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:10:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:10:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:10:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:47.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:47.639882 2089 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:47.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:47.642666 2089 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:47.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:47.678921 2116 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:48.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:48.101181 2108 catalog_manager.cc:1269] Loaded cluster ID: 0566e41626e94c8490bb6934090e836e
14:10:48.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:48.102052 2108 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0: loading cluster ID for follower catalog manager: success
14:10:48.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:48.120680 2108 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0: acquiring CA information for follower catalog manager: success
14:10:48.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:48.126971 2108 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 33fdaeff36e742b0a90625272aa165f0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:10:48.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:46.756877 1817 outbound_call.cc:321] RPC callback for RPC call kudu.master.MasterService.ConnectToMaster -> {remote=127.1.196.60:38475, user_credentials={real_user=slave}} blocked reactor thread for 46928.8us
14:10:49.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:47.699213 2120 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:49.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:49.406730 2118 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1725 milliseconds
14:10:49.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.407323 2089 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:49.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:47.688032 2117 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:49.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.413823 2089 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.427119 2089 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.429092 2089 hybrid_clock.cc:648] HybridClock initialized: now 1777903849429030 us; error 72 us; skew 500 ppm
14:10:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.430581 2089 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:49.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.450498 2089 webserver.cc:492] Webserver started at http://127.1.196.1:35329/ using document root <none> and password file <none>
14:10:49.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.452450 2089 fs_manager.cc:362] Metadata directory not provided
14:10:49.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.453234 2089 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:49.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.454239 2089 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:10:49.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.461865 2089 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-0/data/instance:
14:10:49.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83f99f688e414586a7032f67c348eb88"
14:10:49.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:49 on dist-test-slave-vb5z"
14:10:49.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.464042 2089 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-0/wal/instance:
14:10:49.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83f99f688e414586a7032f67c348eb88"
14:10:49.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:49 on dist-test-slave-vb5z"
14:10:49.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.480882 2089 fs_manager.cc:696] Time spent creating directory manager: real 0.016s user 0.009s sys 0.004s
14:10:49.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.494941 2128 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:49.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.496999 2089 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.007s sys 0.003s
14:10:49.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.497807 2089 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-0/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-0/wal
14:10:49.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83f99f688e414586a7032f67c348eb88"
14:10:49.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:49 on dist-test-slave-vb5z"
14:10:49.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.498672 2089 fs_report.cc:389] FS layout report
14:10:49.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:49.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-0/wal
14:10:49.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-0/wal
14:10:49.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-0/data/data
14:10:49.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:49.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:49.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:49.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:49.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:49.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:49.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:49.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:49.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:49.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:49.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:49.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.598644 2089 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:49.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.601385 2089 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:49.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.602337 2089 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.616357 2089 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:10:49.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.629133 2089 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:10:49.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.629925 2089 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.000s
14:10:49.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.630623 2089 ts_tablet_manager.cc:616] Registered 0 tablets
14:10:49.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:49.632845 2089 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s user 0.000s sys 0.000s
14:10:50.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.239394 2089 rpc_server.cc:307] RPC server started. Bound to: 127.1.196.1:33769
14:10:50.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.239547 2241 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.196.1:33769 every 8 connection(s)
14:10:50.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.242825 2089 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-0/data/info.pb
14:10:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.246948 1808 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 2089
14:10:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.247785 1808 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-0/wal/instance
14:10:50.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.275359 1808 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:10:50.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:10:50.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-1/wal
14:10:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-1/data
14:10:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-1/logs
14:10:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-1/data/info.pb
14:10:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:50.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:50.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:50.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:10:50.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:50.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.196.2:0
14:10:50.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.196.2
14:10:50.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.196.2
14:10:50.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:50.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.196.62:34245,127.1.196.61:40113,127.1.196.60:38475
14:10:50.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.196.20:37005
14:10:50.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:50.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:50.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:10:50.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.324520 2243 heartbeater.cc:344] Connected to a master server at 127.1.196.61:40113
14:10:50.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.325052 2243 heartbeater.cc:461] Registering TS with master...
14:10:50.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.326380 2243 heartbeater.cc:507] Master 127.1.196.61:40113 requested a full tablet report, sending...
14:10:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.329507 2242 heartbeater.cc:344] Connected to a master server at 127.1.196.60:38475
14:10:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.329914 2242 heartbeater.cc:461] Registering TS with master...
14:10:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.330137 2246 heartbeater.cc:344] Connected to a master server at 127.1.196.62:34245
14:10:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.330454 2246 heartbeater.cc:461] Registering TS with master...
14:10:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.330964 2242 heartbeater.cc:507] Master 127.1.196.60:38475 requested a full tablet report, sending...
14:10:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.331481 1928 ts_manager.cc:194] Registered new tserver with Master: 83f99f688e414586a7032f67c348eb88 (127.1.196.1:33769)
14:10:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.331779 2246 heartbeater.cc:507] Master 127.1.196.62:34245 requested a full tablet report, sending...
14:10:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.336185 1853 ts_manager.cc:194] Registered new tserver with Master: 83f99f688e414586a7032f67c348eb88 (127.1.196.1:33769)
14:10:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.338234 2008 ts_manager.cc:194] Registered new tserver with Master: 83f99f688e414586a7032f67c348eb88 (127.1.196.1:33769)
14:10:50.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.341235 2008 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.196.1:46647
14:10:50.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:50.661813 2251 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:10:50.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:50.662529 2251 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:50.663159 2251 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:10:50.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:50.706696 2251 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:10:50.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:50.707774 2251 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.196.2
14:10:50.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.748536 2251 tablet_server_runner.cc:78] Tablet server non-default flags:
14:10:50.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:50.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.196.20:37005
14:10:50.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:50.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:50.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-1/data
14:10:50.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-1/wal
14:10:50.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:50.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:50.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.196.2:0
14:10:50.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:50.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:50.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:50.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-1/data/info.pb
14:10:50.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.196.2
14:10:50.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:50.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.196.62:34245,127.1.196.61:40113,127.1.196.60:38475
14:10:50.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:50.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:50.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:50.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:50.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:50.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.196.2
14:10:50.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-1/logs
14:10:50.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:50.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:50.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:50.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:10:50.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:10:50.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:10:50.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:50.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:10:50.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:10:50.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:50.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.750160 2251 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:50.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:50.751781 2251 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:50.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:50.768357 2258 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:51.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:51.345585 2242 heartbeater.cc:499] Master 127.1.196.60:38475 was elected leader, sending a full tablet report...
14:10:52.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:52.170691 2257 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 2251
14:10:52.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:52.178372 2251 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.411s user 0.471s sys 0.940s
14:10:52.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:50.770431 2259 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:52.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:52.178750 2251 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.412s user 0.471s sys 0.940s
14:10:52.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:52.190104 2262 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:52.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.190228 2251 server_base.cc:1061] running on GCE node
14:10:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.191392 2251 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:52.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.193463 2251 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:52.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.194835 2251 hybrid_clock.cc:648] HybridClock initialized: now 1777903852194799 us; error 40 us; skew 500 ppm
14:10:52.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.195748 2251 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:52.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.202530 2251 webserver.cc:492] Webserver started at http://127.1.196.2:34497/ using document root <none> and password file <none>
14:10:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.203518 2251 fs_manager.cc:362] Metadata directory not provided
14:10:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.203734 2251 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:52.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.204111 2251 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:10:52.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.208628 2251 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-1/data/instance:
14:10:52.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7b16af2166664291b26ffeb0466e1fc5"
14:10:52.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:52 on dist-test-slave-vb5z"
14:10:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.209751 2251 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-1/wal/instance:
14:10:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7b16af2166664291b26ffeb0466e1fc5"
14:10:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:52 on dist-test-slave-vb5z"
14:10:52.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.217857 2251 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.005s
14:10:52.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.224434 2268 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:52.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.225666 2251 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
14:10:52.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.225996 2251 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-1/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-1/wal
14:10:52.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7b16af2166664291b26ffeb0466e1fc5"
14:10:52.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:52 on dist-test-slave-vb5z"
14:10:52.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.226380 2251 fs_report.cc:389] FS layout report
14:10:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-1/wal
14:10:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-1/wal
14:10:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-1/data/data
14:10:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.306762 2251 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:52.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.308416 2251 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:52.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.308995 2251 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:52.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.312888 2251 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:10:52.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.319494 2251 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:10:52.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.319862 2251 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
14:10:52.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.320192 2251 ts_tablet_manager.cc:616] Registered 0 tablets
14:10:52.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.320436 2251 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:10:52.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.480777 2251 rpc_server.cc:307] RPC server started. Bound to: 127.1.196.2:44229
14:10:52.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.480886 2381 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.196.2:44229 every 8 connection(s)
14:10:52.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.483238 2251 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-1/data/info.pb
14:10:52.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.489929 1808 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 2251
14:10:52.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.490402 1808 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-1/wal/instance
14:10:52.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.513266 1808 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:10:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:10:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-2/wal
14:10:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-2/data
14:10:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-2/logs
14:10:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-2/data/info.pb
14:10:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:10:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.196.3:0
14:10:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.196.3
14:10:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.196.3
14:10:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.196.62:34245,127.1.196.61:40113,127.1.196.60:38475
14:10:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.196.20:37005
14:10:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:10:52.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.536909 2382 heartbeater.cc:344] Connected to a master server at 127.1.196.60:38475
14:10:52.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.537505 2382 heartbeater.cc:461] Registering TS with master...
14:10:52.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.538599 2383 heartbeater.cc:344] Connected to a master server at 127.1.196.61:40113
14:10:52.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.538615 2384 heartbeater.cc:344] Connected to a master server at 127.1.196.62:34245
14:10:52.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.539060 2383 heartbeater.cc:461] Registering TS with master...
14:10:52.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.539098 2382 heartbeater.cc:507] Master 127.1.196.60:38475 requested a full tablet report, sending...
14:10:52.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.539393 2384 heartbeater.cc:461] Registering TS with master...
14:10:52.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.540218 2383 heartbeater.cc:507] Master 127.1.196.61:40113 requested a full tablet report, sending...
14:10:52.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.540421 2384 heartbeater.cc:507] Master 127.1.196.62:34245 requested a full tablet report, sending...
14:10:52.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.542749 2008 ts_manager.cc:194] Registered new tserver with Master: 7b16af2166664291b26ffeb0466e1fc5 (127.1.196.2:44229)
14:10:52.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.543860 1853 ts_manager.cc:194] Registered new tserver with Master: 7b16af2166664291b26ffeb0466e1fc5 (127.1.196.2:44229)
14:10:52.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.544236 1928 ts_manager.cc:194] Registered new tserver with Master: 7b16af2166664291b26ffeb0466e1fc5 (127.1.196.2:44229)
14:10:52.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.544910 2008 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.196.2:54503
14:10:52.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:52.853580 2392 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:10:52.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:52.854211 2392 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:52.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:52.854784 2392 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:10:52.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:52.900053 2392 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:10:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:52.901041 2392 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.196.3
14:10:52.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.940707 2392 tablet_server_runner.cc:78] Tablet server non-default flags:
14:10:52.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:52.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.196.20:37005
14:10:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-2/data
14:10:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-2/wal
14:10:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.196.3:0
14:10:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-2/data/info.pb
14:10:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.196.3
14:10:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.196.62:34245,127.1.196.61:40113,127.1.196.60:38475
14:10:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.196.3
14:10:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-2/logs
14:10:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:10:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:10:52.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:10:52.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:52.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:10:52.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:10:52.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:52.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.942617 2392 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:52.945086 2392 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:52.961077 2398 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:53.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:53.549757 2382 heartbeater.cc:499] Master 127.1.196.60:38475 was elected leader, sending a full tablet report...
14:10:54.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:52.964494 2399 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:54.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:52.967537 2401 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:54.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:54.259351 2400 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1296 milliseconds
14:10:54.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.259503 2392 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:54.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.260782 2392 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:54.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.263674 2392 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:54.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.265120 2392 hybrid_clock.cc:648] HybridClock initialized: now 1777903854265076 us; error 58 us; skew 500 ppm
14:10:54.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.265916 2392 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:54.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.274799 2392 webserver.cc:492] Webserver started at http://127.1.196.3:46017/ using document root <none> and password file <none>
14:10:54.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.276074 2392 fs_manager.cc:362] Metadata directory not provided
14:10:54.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.276366 2392 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:54.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.276850 2392 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:10:54.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.281226 2392 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-2/data/instance:
14:10:54.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b9c3625c5b34054b38ba7c980b22b31"
14:10:54.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:54 on dist-test-slave-vb5z"
14:10:54.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.282307 2392 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-2/wal/instance:
14:10:54.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b9c3625c5b34054b38ba7c980b22b31"
14:10:54.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:54 on dist-test-slave-vb5z"
14:10:54.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.289547 2392 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
14:10:54.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.295432 2409 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:54.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.296630 2392 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
14:10:54.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.296983 2392 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-2/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-2/wal
14:10:54.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b9c3625c5b34054b38ba7c980b22b31"
14:10:54.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:54 on dist-test-slave-vb5z"
14:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.297401 2392 fs_report.cc:389] FS layout report
14:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-2/wal
14:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-2/wal
14:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-2/data/data
14:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.355845 2392 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:54.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.357389 2392 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:54.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.357959 2392 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:54.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.360826 2392 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:10:54.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.365629 2392 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:10:54.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.365880 2392 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:10:54.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.366173 2392 ts_tablet_manager.cc:616] Registered 0 tablets
14:10:54.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.366395 2392 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:10:54.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.554307 2392 rpc_server.cc:307] RPC server started. Bound to: 127.1.196.3:41847
14:10:54.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.554415 2521 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.196.3:41847 every 8 connection(s)
14:10:54.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.557614 2392 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-2/data/info.pb
14:10:54.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.566082 1808 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 2392
14:10:54.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.566691 1808 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster16655005156303022836/ts-2/wal/instance
14:10:54.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.605118 2522 heartbeater.cc:344] Connected to a master server at 127.1.196.60:38475
14:10:54.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.605707 2522 heartbeater.cc:461] Registering TS with master...
14:10:54.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.607218 2522 heartbeater.cc:507] Master 127.1.196.60:38475 requested a full tablet report, sending...
14:10:54.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.608047 2524 heartbeater.cc:344] Connected to a master server at 127.1.196.62:34245
14:10:54.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.608410 2524 heartbeater.cc:461] Registering TS with master...
14:10:54.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.608800 2523 heartbeater.cc:344] Connected to a master server at 127.1.196.61:40113
14:10:54.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.609208 2523 heartbeater.cc:461] Registering TS with master...
14:10:54.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.609366 2524 heartbeater.cc:507] Master 127.1.196.62:34245 requested a full tablet report, sending...
14:10:54.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.610369 2523 heartbeater.cc:507] Master 127.1.196.61:40113 requested a full tablet report, sending...
14:10:54.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.612300 1853 ts_manager.cc:194] Registered new tserver with Master: 0b9c3625c5b34054b38ba7c980b22b31 (127.1.196.3:41847)
14:10:54.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.612238 2008 ts_manager.cc:194] Registered new tserver with Master: 0b9c3625c5b34054b38ba7c980b22b31 (127.1.196.3:41847)
14:10:54.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.613579 1928 ts_manager.cc:194] Registered new tserver with Master: 0b9c3625c5b34054b38ba7c980b22b31 (127.1.196.3:41847)
14:10:54.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.614213 2008 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.196.3:56039
14:10:54.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:54.635139 1808 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
14:10:54.646 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:10:54.647 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:10:54.656 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.196.62"
port: 34245
}
bound_http_address {
host: "127.1.196.62"
port: 39549
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.196.61"
port: 40113
}
bound_http_address {
host: "127.1.196.61"
port: 45465
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.196.60"
port: 38475
}
bound_http_address {
host: "127.1.196.60"
port: 41469
}
}
}
14:10:54.659 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:10:54.662 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.1.196.1"
port: 33769
}
bound_http_address {
host: "127.1.196.1"
port: 35329
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.1.196.2"
port: 44229
}
bound_http_address {
host: "127.1.196.2"
port: 34497
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.1.196.3"
port: 41847
}
bound_http_address {
host: "127.1.196.3"
port: 46017
}
}
}
14:10:54.662 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:10:54.829 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
14:10:54.908 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:10:55.031 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskLlQHbR/test-tmp
14:10:55.032 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:10:55.032 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskLlQHbR/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:10:55.036 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
14:10:55.442 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:55.443 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:10:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:55.446455 2534 mini_oidc.cc:150] Starting JWKS server
14:10:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:55.450049 2534 webserver.cc:492] Webserver started at http://127.0.0.1:34683/ using document root <none> and password file <none>
14:10:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:55.450613 2534 mini_oidc.cc:186] Starting OIDC Discovery server
14:10:55.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:55.453039 2534 webserver.cc:492] Webserver started at http://127.0.0.1:41675/ using document root <none> and password file <none>
14:10:55.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
14:10:55.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/principal' for realm 'KRBTEST.COM',
14:10:55.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
14:10:55.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:10:55 dist-test-slave-vb5z krb5kdc[2546](info): setting up network...
14:10:55.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
14:10:55.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:10:55 dist-test-slave-vb5z krb5kdc[2546](info): set up 2 sockets
14:10:55.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:10:55 dist-test-slave-vb5z krb5kdc[2546](info): commencing operation
14:10:55.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
14:10:55.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:55.617892 2522 heartbeater.cc:499] Master 127.1.196.60:38475 was elected leader, sending a full tablet report...
14:10:57.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:57.634006 2534 mini_kdc.cc:121] Time spent starting KDC: real 2.142s user 0.003s sys 0.021s
14:10:57.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
14:10:57.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:10:57.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
14:10:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
14:10:57.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:10:57.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
14:10:57.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
14:10:57.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:10:57.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
14:10:57.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:10:57.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/test-user.keytab.
14:10:57.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/test-user.keytab.
14:10:57.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:10:57 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903857, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:57.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:10:57.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-04T14:10:57Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:10:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-04T14:10:57Z Disabled control of system clock
14:10:57.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.121.190@KRBTEST.COM; defaulting to no policy
14:10:57.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:57.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.121.190@KRBTEST.COM" created.
14:10:57.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:57.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.121.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:10:57.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.121.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:10:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.121.190@KRBTEST.COM; defaulting to no policy
14:10:57.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:57.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.121.190@KRBTEST.COM" created.
14:10:57.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:57.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.121.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:10:57.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.121.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:57.873692 2534 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/logs
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data/info.pb
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:57.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:57.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:57.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:57.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:57.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:57.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.190:34499
14:10:57.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.190
14:10:57.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:57.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:10:57.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.190
14:10:57.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:57.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:57.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:57.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:10:57.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:10:57.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:57.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:57.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:57.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:57.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:10:57.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:57.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:10:57.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:10:57.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:58.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:58.209836 2563 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:10:58.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:58.210455 2563 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:58.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:58.211031 2563 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:10:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:58.245086 2563 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:10:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:58.245399 2563 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:10:58.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:58.245831 2563 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:10:58.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:58.246168 2563 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:58.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:58.246523 2563 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:10:58.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:58.246858 2563 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:58.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:58.247109 2563 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34683/jwks.json
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:58.286458 2563 master_runner.cc:386] Master server non-default flags:
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.190
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.190:34499
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:10:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data/info.pb
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.190
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/logs
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:10:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:58.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:58.288203 2563 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:58.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:58.289606 2563 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:58.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:58.302440 2569 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:59.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:58.305035 2570 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:58.310151 2572 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:59.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:59.620296 2571 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1311 milliseconds
14:10:59.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.620410 2563 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.621652 2563 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:59.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.624545 2563 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:59.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.625941 2563 hybrid_clock.cc:648] HybridClock initialized: now 1777903859625888 us; error 47 us; skew 500 ppm
14:10:59.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.626837 2563 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:59.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:10:59 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903859, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:59.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.634382 2563 init.cc:377] Logged in from keytab as kudu/127.2.121.190@KRBTEST.COM (short username kudu)
14:10:59.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.642143 2563 webserver.cc:492] Webserver started at http://127.2.121.190:38001/ using document root <none> and password file <none>
14:10:59.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.643128 2563 fs_manager.cc:362] Metadata directory not provided
14:10:59.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.643318 2563 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:59.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.643783 2563 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:10:59.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.648428 2563 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data/instance:
14:10:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a33f80b015d34ff687945c6784be125b"
14:10:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:59 on dist-test-slave-vb5z"
14:10:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.649910 2563 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal/instance:
14:10:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a33f80b015d34ff687945c6784be125b"
14:10:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:59 on dist-test-slave-vb5z"
14:10:59.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.658288 2563 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
14:10:59.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.664644 2581 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:59.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.666006 2563 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
14:10:59.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.666424 2563 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:10:59.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a33f80b015d34ff687945c6784be125b"
14:10:59.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:59 on dist-test-slave-vb5z"
14:10:59.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.666890 2563 fs_report.cc:389] FS layout report
14:10:59.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:59.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:10:59.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:10:59.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data/data
14:10:59.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:59.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:59.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:59.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:59.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:59.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:59.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:59.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:59.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:59.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:59.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:59.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.726034 2563 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:59.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.732357 2563 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:59.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.732894 2563 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:59.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.820010 2563 rpc_server.cc:307] RPC server started. Bound to: 127.2.121.190:34499
14:10:59.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.820087 2632 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.121.190:34499 every 8 connection(s)
14:10:59.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.822793 2563 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data/info.pb
14:10:59.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.827788 2534 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 2563
14:10:59.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.828342 2534 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal/instance
14:10:59.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.831029 2633 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:10:59.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.121.189@KRBTEST.COM; defaulting to no policy
14:10:59.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.847018 2633 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } has no permanent_uuid. Determining permanent_uuid...
14:10:59.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:59.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.121.189@KRBTEST.COM" created.
14:10:59.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:10:59 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903859, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.190@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:10:59.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:59.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.121.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:10:59.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.121.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:10:59.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.893810 2633 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } has no permanent_uuid. Determining permanent_uuid...
14:10:59.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:59.896246 2585 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:10:59.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.121.189@KRBTEST.COM; defaulting to no policy
14:10:59.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:59.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.121.189@KRBTEST.COM" created.
14:10:59.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:59.901304 2633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.189:43783: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:10:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.121.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:10:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.121.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:10:59.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.928191 2534 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:10:59.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:10:59.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:10:59.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data
14:10:59.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:59.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:59.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/logs
14:10:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data/info.pb
14:10:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:59.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.189:43783
14:10:59.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.189
14:10:59.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:59.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:10:59.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.189
14:10:59.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:59.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:59.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:59.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:10:59.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:10:59.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:59.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:59.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:59.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:59.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:10:59.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:59.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:10:59.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:10:59.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:59.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:10:59.950819 2633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } attempt: 1
14:10:59.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:10:59.954936 2633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.189:43783: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:00.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:00.023366 2633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } attempt: 2
14:11:00.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:00.028726 2633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.189:43783: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:00.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:00.120180 2633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } attempt: 3
14:11:00.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:00.125052 2633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.189:43783: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:00.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:00.268455 2633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } attempt: 4
14:11:00.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:00.272639 2633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.189:43783: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:00.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:00.274549 2643 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:11:00.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:00.275097 2643 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:00.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:00.275741 2643 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:11:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:00.310266 2643 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:00.310511 2643 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:11:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:00.310890 2643 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:11:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:00.311208 2643 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:00.311551 2643 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:11:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:00.311822 2643 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:00.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:00.311995 2643 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34683/jwks.json
14:11:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:00.352517 2643 master_runner.cc:386] Master server non-default flags:
14:11:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data
14:11:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.189
14:11:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.189:43783
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data/info.pb
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.189
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/logs
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:00.353991 2643 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:00.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:00.355225 2643 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:00.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:00.367344 2649 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:00.572100 2633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } attempt: 5
14:11:00.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:00.577477 2633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.189:43783: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:01.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.124946 2633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } attempt: 6
14:11:01.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:01.131882 2633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.189:43783: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:01.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:00.370062 2650 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:01.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:00.375059 2652 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:01.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:01.647753 2651 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
14:11:01.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.647797 2643 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:11:01.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.651790 2643 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:01.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.654767 2643 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:01.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.656167 2643 hybrid_clock.cc:648] HybridClock initialized: now 1777903861656124 us; error 61 us; skew 500 ppm
14:11:01.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.657042 2643 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:01.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:01 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903861, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:01.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.663779 2643 init.cc:377] Logged in from keytab as kudu/127.2.121.189@KRBTEST.COM (short username kudu)
14:11:01.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.669727 2643 webserver.cc:492] Webserver started at http://127.2.121.189:36363/ using document root <none> and password file <none>
14:11:01.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.670837 2643 fs_manager.cc:362] Metadata directory not provided
14:11:01.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.671095 2643 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:01.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.671792 2643 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:11:01.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.677014 2643 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data/instance:
14:11:01.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5cbbf08fc96d4a45af7a03eec236c560"
14:11:01.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:01 on dist-test-slave-vb5z"
14:11:01.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.678087 2643 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal/instance:
14:11:01.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5cbbf08fc96d4a45af7a03eec236c560"
14:11:01.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:01 on dist-test-slave-vb5z"
14:11:01.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.685858 2643 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.002s sys 0.007s
14:11:01.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.692364 2664 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:01.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.693774 2643 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
14:11:01.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.694165 2643 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:01.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5cbbf08fc96d4a45af7a03eec236c560"
14:11:01.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:01 on dist-test-slave-vb5z"
14:11:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.694499 2643 fs_report.cc:389] FS layout report
14:11:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data/data
14:11:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:01.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.766624 2643 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.772850 2643 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.773362 2643 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:01.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.873337 2643 rpc_server.cc:307] RPC server started. Bound to: 127.2.121.189:43783
14:11:01.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.873402 2715 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.121.189:43783 every 8 connection(s)
14:11:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.876917 2643 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data/info.pb
14:11:01.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.882563 2534 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 2643
14:11:01.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.883193 2534 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal/instance
14:11:01.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.886883 2716 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:11:01.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.121.188@KRBTEST.COM; defaulting to no policy
14:11:01.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:01.908983 2716 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } has no permanent_uuid. Determining permanent_uuid...
14:11:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.121.188@KRBTEST.COM" created.
14:11:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:02 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903861, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.189@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:02.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:02.055243 2716 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } has no permanent_uuid. Determining permanent_uuid...
14:11:02.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:02.192588 2633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } attempt: 7
14:11:02.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:02 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903861, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.189@KRBTEST.COM for kudu/127.2.121.189@KRBTEST.COM
14:11:02.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:02 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903859, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.190@KRBTEST.COM for kudu/127.2.121.189@KRBTEST.COM
14:11:02.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:02.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.121.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:11:02.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.121.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:11:02.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.283342 2534 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.2.121.188: real 0.398s user 0.000s sys 0.014s
14:11:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:02.296249 2716 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } has no permanent_uuid. Determining permanent_uuid...
14:11:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:02.297767 2633 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } has no permanent_uuid. Determining permanent_uuid...
14:11:02.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.299824 2667 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:02.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.121.188@KRBTEST.COM; defaulting to no policy
14:11:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.121.188@KRBTEST.COM" created.
14:11:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.304663 2633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.188:44851: Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:02.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.305826 2716 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.188:44851: Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:02.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:02.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.121.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:11:02.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.121.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:02.327602 2534 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/logs
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data/info.pb
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.188:44851
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.188
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.188
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:02.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:02.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:02.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:02.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:02.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:11:02.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:02.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:02.355312 2716 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } attempt: 1
14:11:02.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.359077 2716 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.188:44851: Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:02.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:02.363119 2633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } attempt: 1
14:11:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.367139 2633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.188:44851: Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:02.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:02.427529 2716 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } attempt: 2
14:11:02.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.432296 2716 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.188:44851: Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:02.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:02.448596 2633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } attempt: 2
14:11:02.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.453408 2633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.188:44851: Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:02.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:02.523782 2716 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } attempt: 3
14:11:02.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.528756 2716 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.188:44851: Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:02.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:02.538877 2633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } attempt: 3
14:11:02.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.543643 2633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.188:44851: Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:02.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:02.672163 2716 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } attempt: 4
14:11:02.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.676442 2716 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.188:44851: Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:02.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.682229 2729 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:11:02.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.682775 2729 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:02.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.683317 2729 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:11:02.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:02.684080 2633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } attempt: 4
14:11:02.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.688414 2633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.188:44851: Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:02.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.717828 2729 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:02.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.718068 2729 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:11:02.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.718334 2729 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:11:02.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.718523 2729 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:02.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.718729 2729 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:11:02.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.718922 2729 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:02.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.719072 2729 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34683/jwks.json
14:11:02.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:02.758852 2729 master_runner.cc:386] Master server non-default flags:
14:11:02.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:02.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:02.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:02.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:02.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data
14:11:02.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:02.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:02.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:02.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:02.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.188
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.188:44851
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data/info.pb
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.188
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/logs
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:11:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:02.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:02.760188 2729 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:02.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:02.761502 2729 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:02.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.776507 2736 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:02.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:02.971895 2633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } attempt: 5
14:11:02.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:02.976074 2716 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } attempt: 5
14:11:02.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.980688 2633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.188:44851: Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:02.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.980932 2716 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.188:44851: Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:03.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:03.528357 2716 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } attempt: 6
14:11:03.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:03.534402 2633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } attempt: 6
14:11:03.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:03.563623 2716 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.188:44851: Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:03.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:03.577971 2633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.188:44851: Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:04.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:04.178864 2735 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 2729
14:11:04.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:04.564131 2738 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1785 milliseconds
14:11:04.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:02.778906 2737 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:04.565563 2739 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:04.562928 2729 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.785s user 0.598s sys 1.136s
14:11:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:04.566236 2729 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.789s user 0.598s sys 1.136s
14:11:04.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.566583 2729 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:11:04.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.571206 2729 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:04.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.574492 2729 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.575878 2729 hybrid_clock.cc:648] HybridClock initialized: now 1777903864575847 us; error 81 us; skew 500 ppm
14:11:04.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.576743 2729 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:04.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:04 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903864, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:04.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.583212 2729 init.cc:377] Logged in from keytab as kudu/127.2.121.188@KRBTEST.COM (short username kudu)
14:11:04.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.589563 2729 webserver.cc:492] Webserver started at http://127.2.121.188:33541/ using document root <none> and password file <none>
14:11:04.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.590595 2729 fs_manager.cc:362] Metadata directory not provided
14:11:04.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.590860 2729 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:04.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.591411 2729 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:11:04.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.596158 2729 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data/instance:
14:11:04.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3e861f07d8954c4fa14d0eb4457d7f28"
14:11:04.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:04 on dist-test-slave-vb5z"
14:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.597429 2729 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal/instance:
14:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3e861f07d8954c4fa14d0eb4457d7f28"
14:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:04 on dist-test-slave-vb5z"
14:11:04.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.606284 2729 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.003s
14:11:04.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.611604 2633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } attempt: 7
14:11:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.612833 2754 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.614466 2729 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
14:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.614895 2729 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3e861f07d8954c4fa14d0eb4457d7f28"
14:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:04 on dist-test-slave-vb5z"
14:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.615321 2729 fs_report.cc:389] FS layout report
14:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data/data
14:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:04.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:04.623091 2633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.188:44851: Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:04.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.624195 2716 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } attempt: 7
14:11:04.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:04.632189 2716 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.121.188:44851: Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:04.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.703399 2729 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.712956 2729 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.713580 2729 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:04.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.806267 2729 rpc_server.cc:307] RPC server started. Bound to: 127.2.121.188:44851
14:11:04.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.806332 2809 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.121.188:44851 every 8 connection(s)
14:11:04.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.809312 2729 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data/info.pb
14:11:04.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.812140 2534 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 2729
14:11:04.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.812691 2534 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal/instance
14:11:04.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.817641 2810 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:11:04.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.835206 2810 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } has no permanent_uuid. Determining permanent_uuid...
14:11:04.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:04 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903857, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:04.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:04 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903864, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.188@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:04.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:04 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903857, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.121.189@KRBTEST.COM
14:11:04.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.877766 2810 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } has no permanent_uuid. Determining permanent_uuid...
14:11:04.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:04 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903864, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.188@KRBTEST.COM for kudu/127.2.121.189@KRBTEST.COM
14:11:04.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:04 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903857, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.121.188@KRBTEST.COM
14:11:04.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.913547 2810 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } has no permanent_uuid. Determining permanent_uuid...
14:11:04.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:04 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903864, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.188@KRBTEST.COM for kudu/127.2.121.188@KRBTEST.COM
14:11:04.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.961805 2810 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: Bootstrap starting.
14:11:04.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.968819 2810 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: Neither blocks nor log segments found. Creating new log.
14:11:04.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.972537 2810 log.cc:826] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: Log is configured to *not* fsync() on all Append() calls
14:11:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:04.977346 2810 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: No bootstrap required, opened a new log
14:11:05.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:05.003576 2810 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:05.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:05.004231 2810 raft_consensus.cc:385] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:11:05.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:05.004423 2810 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3e861f07d8954c4fa14d0eb4457d7f28, State: Initialized, Role: FOLLOWER
14:11:05.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:05.005206 2810 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:05.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:05.008536 2821 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:05.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:05.009166 2821 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: This master's current role is: FOLLOWER
14:11:05.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:05.010311 2810 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: configured and running, proceeding with master startup.
14:11:05.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:05.010458 2821 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:11:05.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:05.010917 2821 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:05.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:05.014004 2821 leader_election.cc:290] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a33f80b015d34ff687945c6784be125b (127.2.121.190:34499), 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783)
14:11:05.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:05.052821 2832 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:11:05.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:05.053246 2832 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:11:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:05.048117 2729 master_runner.cc:186] Error getting master registration for 127.2.121.190:34499: OK, instance_id { permanent_uuid: "a33f80b015d34ff687945c6784be125b" instance_seqno: 1777903859796264 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:11:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:05.083642 2729 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:11:05.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:05.221251 2764 tablet.cc:2404] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:11:05.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:05.712967 2629 debug-util.cc:398] Leaking SignalData structure 0x7b080007d520 after lost signal to thread 2564
14:11:06.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:06.058139 2832 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:11:06.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:06.058522 2832 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:11:06.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:06.695302 2633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } attempt: 8
14:11:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:06.722784 2716 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } attempt: 8
14:11:06.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:06 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903859, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.190@KRBTEST.COM for kudu/127.2.121.188@KRBTEST.COM
14:11:06.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:06 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903861, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.189@KRBTEST.COM for kudu/127.2.121.188@KRBTEST.COM
14:11:06.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:06.795746 2633 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: Bootstrap starting.
14:11:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:06.826148 2716 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: Bootstrap starting.
14:11:06.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:06.835878 2633 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: Neither blocks nor log segments found. Creating new log.
14:11:06.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:06.840193 2633 log.cc:826] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: Log is configured to *not* fsync() on all Append() calls
14:11:06.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:06.851621 2716 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: Neither blocks nor log segments found. Creating new log.
14:11:06.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:06.858201 2716 log.cc:826] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: Log is configured to *not* fsync() on all Append() calls
14:11:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:06.866854 2633 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: No bootstrap required, opened a new log
14:11:06.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:06.906488 2716 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: No bootstrap required, opened a new log
14:11:06.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:06.950415 2633 raft_consensus.cc:359] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:06.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:06.952831 2633 raft_consensus.cc:385] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:11:06.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:06.954008 2633 raft_consensus.cc:740] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a33f80b015d34ff687945c6784be125b, State: Initialized, Role: FOLLOWER
14:11:06.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:06.968063 2633 consensus_queue.cc:260] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:06.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:06.973333 2762 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.2.121.190:34499 timed out after 1.957s (SENT)
14:11:07.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.001187 2841 sys_catalog.cc:455] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:07.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.001996 2841 sys_catalog.cc:458] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: This master's current role is: FOLLOWER
14:11:07.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.003786 2633 sys_catalog.cc:565] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: configured and running, proceeding with master startup.
14:11:07.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.003989 2607 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a33f80b015d34ff687945c6784be125b" is_pre_election: true
14:11:07.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.004935 2607 raft_consensus.cc:2468] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3e861f07d8954c4fa14d0eb4457d7f28 in term 0.
14:11:07.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.006454 2762 leader_election.cc:304] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3e861f07d8954c4fa14d0eb4457d7f28, a33f80b015d34ff687945c6784be125b; no voters:
14:11:07.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.008838 2608 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a33f80b015d34ff687945c6784be125b" is_pre_election: true
14:11:07.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.009500 2608 raft_consensus.cc:2468] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3e861f07d8954c4fa14d0eb4457d7f28 in term 0.
14:11:07.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:07.010118 2608 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:46486 (request call id 1) took 1995 ms (client timeout 1957 ms). Trace:
14:11:07.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:07.010504 2608 rpcz_store.cc:269] 0504 14:11:05.014832 (+ 0us) service_pool.cc:167] Inserting onto call queue
14:11:07.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:05.015010 (+ 178us) service_pool.cc:224] Handling call
14:11:07.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:07.010021 (+1995011us) inbound_call.cc:177] Queueing success response
14:11:07.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:11:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.001756 2716 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:07.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.021381 2716 raft_consensus.cc:385] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:11:07.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.022346 2716 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5cbbf08fc96d4a45af7a03eec236c560, State: Initialized, Role: FOLLOWER
14:11:07.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.024847 2716 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:07.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.029551 2844 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:11:07.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.037515 2847 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:07.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.049909 2844 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:11:07.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.050625 2844 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 0 FOLLOWER]: Advancing to term 1
14:11:07.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.051210 2847 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: This master's current role is: FOLLOWER
14:11:07.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.042678 2716 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: configured and running, proceeding with master startup.
14:11:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.042850 2690 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5cbbf08fc96d4a45af7a03eec236c560" is_pre_election: true
14:11:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.055437 2690 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3e861f07d8954c4fa14d0eb4457d7f28 in term 0.
14:11:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:07.056062 2690 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:41242 (request call id 1) took 2041 ms (client timeout 1957 ms). Trace:
14:11:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:07.056514 2690 rpcz_store.cc:269] 0504 14:11:05.014797 (+ 0us) service_pool.cc:167] Inserting onto call queue
14:11:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:05.015009 (+ 212us) service_pool.cc:224] Handling call
14:11:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:07.055978 (+2040969us) inbound_call.cc:177] Queueing success response
14:11:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:11:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.043525 2691 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5cbbf08fc96d4a45af7a03eec236c560" is_pre_election: true
14:11:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.057650 2691 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3e861f07d8954c4fa14d0eb4457d7f28 in term 0.
14:11:07.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.070320 2844 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:07.081457 2832 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:11:07.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:07.082121 2832 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:11:07.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.086851 2608 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a33f80b015d34ff687945c6784be125b"
14:11:07.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.087484 2608 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 0 FOLLOWER]: Advancing to term 1
14:11:07.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.096765 2608 raft_consensus.cc:2468] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3e861f07d8954c4fa14d0eb4457d7f28 in term 1.
14:11:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.098274 2762 leader_election.cc:304] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3e861f07d8954c4fa14d0eb4457d7f28, a33f80b015d34ff687945c6784be125b; no voters:
14:11:07.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.101467 2844 leader_election.cc:290] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [CANDIDATE]: Term 1 election: Requested vote from peers a33f80b015d34ff687945c6784be125b (127.2.121.190:34499), 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783)
14:11:07.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.102074 2844 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 1 FOLLOWER]: Leader election won for term 1
14:11:07.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.102578 2844 raft_consensus.cc:697] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 1 LEADER]: Becoming Leader. State: Replica: 3e861f07d8954c4fa14d0eb4457d7f28, State: Running, Role: LEADER
14:11:07.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.110682 2858 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:11:07.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.103412 2844 consensus_queue.cc:237] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:07.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.118348 2691 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5cbbf08fc96d4a45af7a03eec236c560"
14:11:07.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.118932 2691 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [term 0 FOLLOWER]: Advancing to term 1
14:11:07.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.148707 2691 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3e861f07d8954c4fa14d0eb4457d7f28 in term 1.
14:11:07.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.166296 2857 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3e861f07d8954c4fa14d0eb4457d7f28. Latest consensus state: current_term: 1 leader_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:07.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.167651 2857 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: This master's current role is: LEADER
14:11:07.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.180949 2862 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
14:11:07.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.127556 2563 master_runner.cc:186] Error getting master registration for 127.2.121.189:43783: OK, instance_id { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" instance_seqno: 1777903861846950 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:11:07.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.201191 2862 catalog_manager.cc:1494] Initializing Kudu cluster ID...
14:11:07.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.211971 2563 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:11:07.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:07.246451 2587 tablet.cc:2404] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:11:07.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.250777 2691 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [term 1 FOLLOWER]: Refusing update from remote peer 3e861f07d8954c4fa14d0eb4457d7f28: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:11:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.253230 2857 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:11:07.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:07.265878 2870 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:11:07.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:07.266285 2870 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:11:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.273514 2608 raft_consensus.cc:1275] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 1 FOLLOWER]: Refusing update from remote peer 3e861f07d8954c4fa14d0eb4457d7f28: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:11:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.275775 2857 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:11:07.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:07.286974 2670 tablet.cc:2404] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:11:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.307849 2841 sys_catalog.cc:455] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3e861f07d8954c4fa14d0eb4457d7f28. Latest consensus state: current_term: 1 leader_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.313268 2868 mvcc.cc:204] Tried to move back new op lower bound from 7282294240188272640 to 7282294239716741120. Current Snapshot: MvccSnapshot[applied={T|T < 7282294240188272640}]
14:11:07.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.313555 2841 sys_catalog.cc:458] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: This master's current role is: FOLLOWER
14:11:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.321709 2873 mvcc.cc:204] Tried to move back new op lower bound from 7282294240188272640 to 7282294239716741120. Current Snapshot: MvccSnapshot[applied={T|T < 7282294240188272640}]
14:11:07.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.325109 2841 sys_catalog.cc:455] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.325923 2841 sys_catalog.cc:458] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: This master's current role is: FOLLOWER
14:11:07.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.333786 2844 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:07.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.334619 2844 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: This master's current role is: LEADER
14:11:07.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.340319 2862 catalog_manager.cc:1357] Generated new cluster ID: 646f046dc1e24a52a10b732269305767
14:11:07.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.340651 2862 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
14:11:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.348510 2847 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3e861f07d8954c4fa14d0eb4457d7f28. Latest consensus state: current_term: 1 leader_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:07.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.350142 2847 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: This master's current role is: FOLLOWER
14:11:07.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.356245 2857 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:07.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.357015 2857 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: This master's current role is: LEADER
14:11:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.365450 2872 mvcc.cc:204] Tried to move back new op lower bound from 7282294240188272640 to 7282294239716741120. Current Snapshot: MvccSnapshot[applied={T|T < 7282294240188272640}]
14:11:07.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.369513 2847 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:07.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.370229 2847 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: This master's current role is: FOLLOWER
14:11:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.419541 2862 catalog_manager.cc:1380] Generated new certificate authority record
14:11:07.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.427214 2862 catalog_manager.cc:1514] Loading token signing keys...
14:11:07.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.557665 2862 catalog_manager.cc:6044] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: Generated new TSK 0
14:11:07.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.560134 2862 catalog_manager.cc:1524] Initializing in-progress tserver states...
14:11:07.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.121.129@KRBTEST.COM; defaulting to no policy
14:11:07.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:07.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.121.129@KRBTEST.COM" created.
14:11:07.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:07.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.121.129 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:11:07.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.121.129 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:11:07.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.121.129@KRBTEST.COM; defaulting to no policy
14:11:07.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:07.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.121.129@KRBTEST.COM" created.
14:11:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:07.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.121.129 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:11:07.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.121.129 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:11:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:07.754299 2534 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-0/wal
14:11:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-0/data
14:11:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-0/logs
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-0/data/info.pb
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.129:0
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.121.129
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.129
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.129
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:08.128515 2858 catalog_manager.cc:1269] Loaded cluster ID: 646f046dc1e24a52a10b732269305767
14:11:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:08.128894 2858 catalog_manager.cc:1562] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: loading cluster ID for follower catalog manager: success
14:11:08.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:08.151494 2858 catalog_manager.cc:1584] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: acquiring CA information for follower catalog manager: success
14:11:08.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:08.160935 2858 catalog_manager.cc:1612] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:08.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:08.274216 2870 catalog_manager.cc:1269] Loaded cluster ID: 646f046dc1e24a52a10b732269305767
14:11:08.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:08.274995 2870 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: loading cluster ID for follower catalog manager: success
14:11:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:08.282867 2870 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: acquiring CA information for follower catalog manager: success
14:11:08.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:08.289011 2870 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:08.380609 2885 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:08.381320 2885 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:08.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:08.382266 2885 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:11:08.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:08.445535 2885 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:11:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:08.447139 2885 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.121.129
14:11:08.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:08.517488 2885 tablet_server_runner.cc:78] Tablet server non-default flags:
14:11:08.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:08.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:08.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:08.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:08.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-0/data
14:11:08.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-0/wal
14:11:08.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:08.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.129
14:11:08.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:08.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:08.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.129:0
14:11:08.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-0/data/info.pb
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.129
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.121.129
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-0/logs
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:08.519541 2885 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:08.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:08.521124 2885 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:08.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:08.596464 2894 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:09.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:09 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903869, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:09.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:09.641960 2577 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:09.695628 2629 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae380 after lost signal to thread 2564
14:11:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:09.696300 2629 debug-util.cc:398] Leaking SignalData structure 0x7b0800075da0 after lost signal to thread 2632
14:11:10.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:09.953819 2892 debug-util.cc:398] Leaking SignalData structure 0x7b0800027120 after lost signal to thread 2885
14:11:10.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:10.970422 2892 kernel_stack_watchdog.cc:198] Thread 2885 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 359ms:
14:11:10.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:11:10.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:11:10.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:11:10.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:11:10.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:08.600006 2893 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:10.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:10.977943 2885 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.385s user 0.599s sys 1.080s
14:11:10.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:10.978353 2885 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.386s user 0.599s sys 1.080s
14:11:10.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:10.996747 2885 server_base.cc:1061] running on GCE node
14:11:10.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:10.997880 2902 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:11.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.003286 2885 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:11.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.024034 2885 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:11.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.025691 2885 hybrid_clock.cc:648] HybridClock initialized: now 1777903871025642 us; error 59 us; skew 500 ppm
14:11:11.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.028156 2885 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:11.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:11 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903871, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.129@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:11.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.037096 2885 init.cc:377] Logged in from keytab as kudu/127.2.121.129@KRBTEST.COM (short username kudu)
14:11:11.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.044420 2885 webserver.cc:492] Webserver started at http://127.2.121.129:34917/ using document root <none> and password file <none>
14:11:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.046047 2885 fs_manager.cc:362] Metadata directory not provided
14:11:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.046350 2885 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.046917 2885 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:11:11.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.054490 2885 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-0/data/instance:
14:11:11.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "94af139f04f64b0891ae8e49f5d705fa"
14:11:11.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:11 on dist-test-slave-vb5z"
14:11:11.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.056690 2885 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-0/wal/instance:
14:11:11.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "94af139f04f64b0891ae8e49f5d705fa"
14:11:11.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:11 on dist-test-slave-vb5z"
14:11:11.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.067523 2885 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.008s sys 0.002s
14:11:11.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.076689 2908 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:11.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.078233 2885 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
14:11:11.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.078658 2885 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-0/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-0/wal
14:11:11.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "94af139f04f64b0891ae8e49f5d705fa"
14:11:11.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:11 on dist-test-slave-vb5z"
14:11:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.079142 2885 fs_report.cc:389] FS layout report
14:11:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-0/wal
14:11:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-0/wal
14:11:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-0/data/data
14:11:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:11.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.148730 2885 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:11.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.156545 2885 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:11.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.157200 2885 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:11.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.162901 2885 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:11:11.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.168767 2885 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:11:11.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.169067 2885 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:11:11.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.169350 2885 ts_tablet_manager.cc:616] Registered 0 tablets
14:11:11.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.169560 2885 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:11:11.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.394295 2885 rpc_server.cc:307] RPC server started. Bound to: 127.2.121.129:38867
14:11:11.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.394410 3020 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.121.129:38867 every 8 connection(s)
14:11:11.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.397035 2885 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-0/data/info.pb
14:11:11.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.403034 2534 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 2885
14:11:11.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.403544 2534 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-0/wal/instance
14:11:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.121.130@KRBTEST.COM; defaulting to no policy
14:11:11.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:11.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.121.130@KRBTEST.COM" created.
14:11:11.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:11 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903871, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.129@KRBTEST.COM for kudu/127.2.121.189@KRBTEST.COM
14:11:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:11 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903871, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.129@KRBTEST.COM for kudu/127.2.121.188@KRBTEST.COM
14:11:11.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:11 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903871, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.129@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.513235 3021 heartbeater.cc:344] Connected to a master server at 127.2.121.188:44851
14:11:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.513775 3021 heartbeater.cc:461] Registering TS with master...
14:11:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.515362 3021 heartbeater.cc:507] Master 127.2.121.188:44851 requested a full tablet report, sending...
14:11:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.518289 3023 heartbeater.cc:344] Connected to a master server at 127.2.121.190:34499
14:11:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.518666 3023 heartbeater.cc:461] Registering TS with master...
14:11:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.518759 2775 ts_manager.cc:194] Registered new tserver with Master: 94af139f04f64b0891ae8e49f5d705fa (127.2.121.129:38867)
14:11:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.519763 3023 heartbeater.cc:507] Master 127.2.121.190:34499 requested a full tablet report, sending...
14:11:11.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.521493 3022 heartbeater.cc:344] Connected to a master server at 127.2.121.189:43783
14:11:11.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.521551 2775 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.2.121.129@KRBTEST.COM'} at 127.2.121.129:48123
14:11:11.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.521888 3022 heartbeater.cc:461] Registering TS with master...
14:11:11.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.522873 3022 heartbeater.cc:507] Master 127.2.121.189:43783 requested a full tablet report, sending...
14:11:11.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.523594 2598 ts_manager.cc:194] Registered new tserver with Master: 94af139f04f64b0891ae8e49f5d705fa (127.2.121.129:38867)
14:11:11.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.526144 2681 ts_manager.cc:194] Registered new tserver with Master: 94af139f04f64b0891ae8e49f5d705fa (127.2.121.129:38867)
14:11:11.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:11.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.121.130 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:11:11.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.121.130 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:11:11.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:11.539678 2534 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.2.121.130: real 0.117s user 0.010s sys 0.006s
14:11:11.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.121.130@KRBTEST.COM; defaulting to no policy
14:11:11.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:11.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.121.130@KRBTEST.COM" created.
14:11:11.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:11.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.121.130 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:11:11.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.121.130 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:11:11.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.595259 2534 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:11.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-1/wal
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-1/data
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-1/logs
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-1/data/info.pb
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.130:0
14:11:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.121.130
14:11:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.130
14:11:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:11:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.130
14:11:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:11:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:11 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903871, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:11.669190 2660 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:11.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:11.986913 3035 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:11:11.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:11.987605 3035 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:11.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:11.988380 3035 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:11:12.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:12.026124 3035 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:11:12.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:12.027180 3035 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.121.130
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:12.067725 3035 tablet_server_runner.cc:78] Tablet server non-default flags:
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-1/data
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-1/wal
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.130
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.130:0
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-1/data/info.pb
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.130
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.121.130
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-1/logs
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:11:12.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:11:12.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:11:12.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:12.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:11:12.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:11:12.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:12.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:12.068969 3035 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:12.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:12.070035 3035 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:12.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:12.083256 3041 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:12.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:12.530330 3021 heartbeater.cc:499] Master 127.2.121.188:44851 was elected leader, sending a full tablet report...
14:11:13.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:12.085479 3042 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:13.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:12.090225 3044 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:13.461596 3043 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1372 milliseconds
14:11:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.461861 3035 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:11:13.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.463323 3035 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:13.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.466861 3035 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:13.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.468355 3035 hybrid_clock.cc:648] HybridClock initialized: now 1777903873468264 us; error 106 us; skew 500 ppm
14:11:13.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.469314 3035 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:13.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:13 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903873, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.130@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.476985 3035 init.cc:377] Logged in from keytab as kudu/127.2.121.130@KRBTEST.COM (short username kudu)
14:11:13.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.483158 3035 webserver.cc:492] Webserver started at http://127.2.121.130:42115/ using document root <none> and password file <none>
14:11:13.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.484689 3035 fs_manager.cc:362] Metadata directory not provided
14:11:13.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.484913 3035 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:13.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.485316 3035 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:11:13.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.491034 3035 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-1/data/instance:
14:11:13.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f94afdf18ee48769d301f9bc00ff0c9"
14:11:13.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:13 on dist-test-slave-vb5z"
14:11:13.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.492666 3035 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-1/wal/instance:
14:11:13.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f94afdf18ee48769d301f9bc00ff0c9"
14:11:13.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:13 on dist-test-slave-vb5z"
14:11:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.500438 3035 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.003s sys 0.004s
14:11:13.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.506624 3055 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.507932 3035 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
14:11:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.508302 3035 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-1/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-1/wal
14:11:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f94afdf18ee48769d301f9bc00ff0c9"
14:11:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:13 on dist-test-slave-vb5z"
14:11:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.508716 3035 fs_report.cc:389] FS layout report
14:11:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-1/wal
14:11:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-1/wal
14:11:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-1/data/data
14:11:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:13.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.581523 3035 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:13.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.589988 3035 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:13.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.590615 3035 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:13.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.593650 3035 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:11:13.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.599090 3035 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:11:13.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.599373 3035 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:11:13.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.599725 3035 ts_tablet_manager.cc:616] Registered 0 tablets
14:11:13.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.599937 3035 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:11:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.794872 3035 rpc_server.cc:307] RPC server started. Bound to: 127.2.121.130:34419
14:11:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.795147 3167 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.121.130:34419 every 8 connection(s)
14:11:13.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.797858 3035 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-1/data/info.pb
14:11:13.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.807863 2534 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 3035
14:11:13.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.808485 2534 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-1/wal/instance
14:11:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.121.131@KRBTEST.COM; defaulting to no policy
14:11:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:13 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903873, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.130@KRBTEST.COM for kudu/127.2.121.189@KRBTEST.COM
14:11:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.121.131@KRBTEST.COM" created.
14:11:13.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:13 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903873, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.130@KRBTEST.COM for kudu/127.2.121.188@KRBTEST.COM
14:11:13.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.932688 3169 heartbeater.cc:344] Connected to a master server at 127.2.121.189:43783
14:11:13.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.933296 3169 heartbeater.cc:461] Registering TS with master...
14:11:13.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.934762 3169 heartbeater.cc:507] Master 127.2.121.189:43783 requested a full tablet report, sending...
14:11:13.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.938618 2681 ts_manager.cc:194] Registered new tserver with Master: 9f94afdf18ee48769d301f9bc00ff0c9 (127.2.121.130:34419)
14:11:13.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:13 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903873, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.130@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:13.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.953269 3168 heartbeater.cc:344] Connected to a master server at 127.2.121.188:44851
14:11:13.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.953672 3168 heartbeater.cc:461] Registering TS with master...
14:11:13.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.954696 3168 heartbeater.cc:507] Master 127.2.121.188:44851 requested a full tablet report, sending...
14:11:13.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.957070 2775 ts_manager.cc:194] Registered new tserver with Master: 9f94afdf18ee48769d301f9bc00ff0c9 (127.2.121.130:34419)
14:11:13.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.959062 2775 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.2.121.130@KRBTEST.COM'} at 127.2.121.130:48651
14:11:13.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:13.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.121.131 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:11:13.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.121.131 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:11:13.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:13.977344 2534 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.2.121.131: real 0.147s user 0.000s sys 0.017s
14:11:13.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.982331 3170 heartbeater.cc:344] Connected to a master server at 127.2.121.190:34499
14:11:13.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.982784 3170 heartbeater.cc:461] Registering TS with master...
14:11:13.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.983525 3170 heartbeater.cc:507] Master 127.2.121.190:34499 requested a full tablet report, sending...
14:11:13.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:13.986303 2598 ts_manager.cc:194] Registered new tserver with Master: 9f94afdf18ee48769d301f9bc00ff0c9 (127.2.121.130:34419)
14:11:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.121.131@KRBTEST.COM; defaulting to no policy
14:11:13.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:13.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.121.131@KRBTEST.COM" created.
14:11:14.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:14.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.121.131 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:11:14.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.121.131 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab.
14:11:14.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:14.028982 2534 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:14.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:14.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-2/wal
14:11:14.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-2/data
14:11:14.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:14.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:14.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:14.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:14.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-2/logs
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-2/data/info.pb
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.131:0
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.121.131
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.131
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.131
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:14.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:14.382330 3182 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:11:14.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:14.382822 3182 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:14.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:14.383453 3182 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:11:14.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:14.423805 3182 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:11:14.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:14.424755 3182 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.121.131
14:11:14.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:14.469920 3182 tablet_server_runner.cc:78] Tablet server non-default flags:
14:11:14.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:14.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:14.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:14.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:14.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-2/data
14:11:14.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-2/wal
14:11:14.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:14.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.131
14:11:14.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:14.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.131:0
14:11:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-2/data/info.pb
14:11:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.131
14:11:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:11:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.121.131
14:11:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-2/logs
14:11:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:11:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:11:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:11:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:11:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:11:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:14.471386 3182 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:14.472443 3182 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:14.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:14.487697 3189 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:14.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:14 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903874, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:14.589586 2750 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:14.963963 3168 heartbeater.cc:499] Master 127.2.121.188:44851 was elected leader, sending a full tablet report...
14:11:16.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:15.888936 3188 debug-util.cc:398] Leaking SignalData structure 0x7b0800027120 after lost signal to thread 3182
14:11:16.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:16.004712 3182 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.515s user 0.558s sys 0.897s
14:11:16.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:16.005136 3182 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.516s user 0.558s sys 0.897s
14:11:16.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:14.491004 3190 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:16.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:16.022323 3196 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:16.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.022483 3182 server_base.cc:1061] running on GCE node
14:11:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.024078 3182 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:16.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.027138 3182 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:16.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.028544 3182 hybrid_clock.cc:648] HybridClock initialized: now 1777903876028509 us; error 63 us; skew 500 ppm
14:11:16.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.029449 3182 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:16.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:16 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903876, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.131@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:16.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.044003 3182 init.cc:377] Logged in from keytab as kudu/127.2.121.131@KRBTEST.COM (short username kudu)
14:11:16.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.049918 3182 webserver.cc:492] Webserver started at http://127.2.121.131:39913/ using document root <none> and password file <none>
14:11:16.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.051172 3182 fs_manager.cc:362] Metadata directory not provided
14:11:16.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.051448 3182 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:16.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.052037 3182 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:11:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.056511 3182 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-2/data/instance:
14:11:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "638f763f19db44b8b70928087de3c6c4"
14:11:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:16 on dist-test-slave-vb5z"
14:11:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.057696 3182 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-2/wal/instance:
14:11:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "638f763f19db44b8b70928087de3c6c4"
14:11:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:16 on dist-test-slave-vb5z"
14:11:16.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.065200 3182 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
14:11:16.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.071241 3202 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:16.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.072492 3182 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
14:11:16.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.072889 3182 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-2/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-2/wal
14:11:16.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "638f763f19db44b8b70928087de3c6c4"
14:11:16.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:16 on dist-test-slave-vb5z"
14:11:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.073293 3182 fs_report.cc:389] FS layout report
14:11:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-2/wal
14:11:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-2/wal
14:11:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-2/data/data
14:11:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:16.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:16.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:16.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.189397 3182 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:16.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.195219 3182 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:16.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.195791 3182 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:16.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.198659 3182 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:11:16.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.203984 3182 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:11:16.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.204228 3182 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:11:16.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.204528 3182 ts_tablet_manager.cc:616] Registered 0 tablets
14:11:16.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.204747 3182 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:11:16.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.399936 3182 rpc_server.cc:307] RPC server started. Bound to: 127.2.121.131:42729
14:11:16.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.400034 3314 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.121.131:42729 every 8 connection(s)
14:11:16.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.402750 3182 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-2/data/info.pb
14:11:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.406025 2534 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 3182
14:11:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.406724 2534 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/ts-2/wal/instance
14:11:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:16 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903876, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.131@KRBTEST.COM for kudu/127.2.121.189@KRBTEST.COM
14:11:16.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:16 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903876, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.131@KRBTEST.COM for kudu/127.2.121.188@KRBTEST.COM
14:11:16.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:16 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903876, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.131@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:16.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.466861 3316 heartbeater.cc:344] Connected to a master server at 127.2.121.189:43783
14:11:16.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.467392 3316 heartbeater.cc:461] Registering TS with master...
14:11:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.468921 3316 heartbeater.cc:507] Master 127.2.121.189:43783 requested a full tablet report, sending...
14:11:16.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.472086 3315 heartbeater.cc:344] Connected to a master server at 127.2.121.188:44851
14:11:16.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.472451 3315 heartbeater.cc:461] Registering TS with master...
14:11:16.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.473459 2681 ts_manager.cc:194] Registered new tserver with Master: 638f763f19db44b8b70928087de3c6c4 (127.2.121.131:42729)
14:11:16.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.473491 3315 heartbeater.cc:507] Master 127.2.121.188:44851 requested a full tablet report, sending...
14:11:16.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.475859 2775 ts_manager.cc:194] Registered new tserver with Master: 638f763f19db44b8b70928087de3c6c4 (127.2.121.131:42729)
14:11:16.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.477296 3317 heartbeater.cc:344] Connected to a master server at 127.2.121.190:34499
14:11:16.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.477638 3317 heartbeater.cc:461] Registering TS with master...
14:11:16.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.477696 2775 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.2.121.131@KRBTEST.COM'} at 127.2.121.131:44135
14:11:16.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.478605 3317 heartbeater.cc:507] Master 127.2.121.190:34499 requested a full tablet report, sending...
14:11:16.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.481077 2598 ts_manager.cc:194] Registered new tserver with Master: 638f763f19db44b8b70928087de3c6c4 (127.2.121.131:42729)
14:11:16.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.491223 2534 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
14:11:16.503 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:16.504 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
14:11:16.512 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
14:11:16.513 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
14:11:16.517 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.2.121.190"
port: 34499
}
bound_http_address {
host: "127.2.121.190"
port: 38001
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.2.121.189"
port: 43783
}
bound_http_address {
host: "127.2.121.189"
port: 36363
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.2.121.188"
port: 44851
}
bound_http_address {
host: "127.2.121.188"
port: 33541
}
}
}
14:11:16.517 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:11:16.519 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.2.121.129"
port: 38867
}
bound_http_address {
host: "127.2.121.129"
port: 34917
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.2.121.130"
port: 34419
}
bound_http_address {
host: "127.2.121.130"
port: 42115
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.2.121.131"
port: 42729
}
bound_http_address {
host: "127.2.121.131"
port: 39913
}
}
}
14:11:16.519 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:11:16.520 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:11:16.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:16 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903876, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:16.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:11:16.584 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:16.587 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc
14:11:16.596 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:11:16.597 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc
14:11:16.599 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:11:16.640 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
14:11:16.642 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:11:16.643 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:11:16.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:16 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903876, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:16.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:11:16.675 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:16.676 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc
14:11:16.677 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:11:16.677 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.121.190:34499
14:11:16.678 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:11:16.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.679435 2534 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 2563
14:11:16.713 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:16.713 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.121.188:44851
14:11:16.713 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:11:16.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.713733 2534 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 2729
14:11:16.744 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:16.744 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.121.189:43783
14:11:16.744 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:11:16.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.744941 2534 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 2643
14:11:16.775 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:16.776 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.121.190:34499
14:11:16.780 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:11:16.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:16.781791 2534 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:16.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:16.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:11:16.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data
14:11:16.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:16.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:16.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:16.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:16.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:16.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:16.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/logs
14:11:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data/info.pb
14:11:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.190:34499
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.190
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38001
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.190
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:16.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:16.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:16.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:11:16.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:16.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:16.963234 3169 heartbeater.cc:646] Failed to heartbeat to 127.2.121.189:43783 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:17.143603 3330 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:11:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:17.144161 3330 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:17.144773 3330 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:11:17.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:17.180224 3330 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:17.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:17.180485 3330 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:11:17.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:17.180766 3330 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:11:17.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:17.180967 3330 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:17.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:17.181162 3330 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:11:17.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:17.181360 3330 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:17.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:17.181517 3330 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34683/jwks.json
14:11:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:17.222872 3330 master_runner.cc:386] Master server non-default flags:
14:11:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data
14:11:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:11:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.190
14:11:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.190:34499
14:11:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data/info.pb
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.190
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38001
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/logs
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:11:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:11:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:11:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:11:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:17.224227 3330 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:17.225275 3330 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:17.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:17.237811 3337 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:17.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:17.481371 3315 heartbeater.cc:499] Master 127.2.121.188:44851 was elected leader, sending a full tablet report...
14:11:17.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:17.502753 3315 heartbeater.cc:646] Failed to heartbeat to 127.2.121.188:44851 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:17.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:17.583489 3021 heartbeater.cc:646] Failed to heartbeat to 127.2.121.188:44851 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:18.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:17.239118 3338 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:18.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:17.240825 3340 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:18.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:18.641745 3339 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1401 milliseconds
14:11:18.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.641856 3330 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:11:18.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.643162 3330 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:18.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.646039 3330 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:18.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.647406 3330 hybrid_clock.cc:648] HybridClock initialized: now 1777903878647372 us; error 51 us; skew 500 ppm
14:11:18.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.648308 3330 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:18.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:18 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903878, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:18.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.654841 3330 init.cc:377] Logged in from keytab as kudu/127.2.121.190@KRBTEST.COM (short username kudu)
14:11:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.660467 3330 webserver.cc:492] Webserver started at http://127.2.121.190:38001/ using document root <none> and password file <none>
14:11:18.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.661494 3330 fs_manager.cc:362] Metadata directory not provided
14:11:18.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.661793 3330 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:18.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.670624 3330 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.002s sys 0.004s
14:11:18.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.675328 3359 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:18.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.676512 3330 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
14:11:18.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.676928 3330 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:11:18.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a33f80b015d34ff687945c6784be125b"
14:11:18.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:59 on dist-test-slave-vb5z"
14:11:18.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.678943 3330 fs_report.cc:389] FS layout report
14:11:18.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:18.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:11:18.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:11:18.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data/data
14:11:18.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:18.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:18.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:18.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:18.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:18.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:18.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:18.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:18.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:18.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:18.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:18.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.739054 3330 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:18.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.750185 3330 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:18.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.750810 3330 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.832116 3330 rpc_server.cc:307] RPC server started. Bound to: 127.2.121.190:34499
14:11:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.832173 3410 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.121.190:34499 every 8 connection(s)
14:11:18.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.835246 3330 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data/info.pb
14:11:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.844339 2534 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 3330
14:11:18.845 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:18.845 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.121.188:44851
14:11:18.845 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:11:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.846849 2534 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data
14:11:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/logs
14:11:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data/info.pb
14:11:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.188:44851
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.188
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33541
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.188
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:18.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:11:18.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:18.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.848526 3411 sys_catalog.cc:263] Verifying existing consensus state
14:11:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.854811 3411 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: Bootstrap starting.
14:11:18.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.906395 3411 log.cc:826] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: Log is configured to *not* fsync() on all Append() calls
14:11:18.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.927673 3411 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:11:18.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.928646 3411 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: Bootstrap complete.
14:11:18.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.958406 3411 raft_consensus.cc:359] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:18.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.961095 3411 raft_consensus.cc:740] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: a33f80b015d34ff687945c6784be125b, State: Initialized, Role: FOLLOWER
14:11:18.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.962257 3411 consensus_queue.cc:260] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:18.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.965268 3415 sys_catalog.cc:455] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:18.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.966339 3415 sys_catalog.cc:458] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: This master's current role is: FOLLOWER
14:11:18.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:18.967334 3411 sys_catalog.cc:565] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: configured and running, proceeding with master startup.
14:11:19.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:19.029909 3429 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:11:19.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:19.050781 3170 heartbeater.cc:344] Connected to a master server at 127.2.121.190:34499
14:11:19.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:19 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903878, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.190@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:19.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:19.058298 3376 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "9f94afdf18ee48769d301f9bc00ff0c9" instance_seqno: 1777903873751575) as {username='kudu', principal='kudu/127.2.121.130@KRBTEST.COM'} at 127.2.121.130:46737; Asking this server to re-register.
14:11:19.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:19.060154 3170 heartbeater.cc:461] Registering TS with master...
14:11:19.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:19.063846 3376 ts_manager.cc:194] Registered new tserver with Master: 9f94afdf18ee48769d301f9bc00ff0c9 (127.2.121.130:34419)
14:11:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:19.070039 3330 master_runner.cc:186] Error getting master registration for 127.2.121.189:43783: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111),
14:11:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:19.087352 3330 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:11:19.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:19.233400 3413 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:11:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:19.234127 3413 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:19.234885 3413 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:11:19.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:19.272558 3413 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:19.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:19.273006 3413 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:11:19.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:19.273352 3413 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:11:19.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:19.273581 3413 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:19.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:19.273794 3413 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:11:19.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:19.273983 3413 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:19.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:19.274119 3413 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34683/jwks.json
14:11:19.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:19.314863 3413 master_runner.cc:386] Master server non-default flags:
14:11:19.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:19.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:19.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:19.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:19.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data
14:11:19.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:19.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.188
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.188:44851
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data/info.pb
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.188
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33541
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/logs
14:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:19.316846 3413 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:19.318197 3413 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:19.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:19.330309 3437 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:19.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:19.559715 3317 heartbeater.cc:344] Connected to a master server at 127.2.121.190:34499
14:11:19.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:19.562433 3376 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "638f763f19db44b8b70928087de3c6c4" instance_seqno: 1777903876359054) as {username='kudu', principal='kudu/127.2.121.131@KRBTEST.COM'} at 127.2.121.131:59711; Asking this server to re-register.
14:11:19.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:19.565054 3317 heartbeater.cc:461] Registering TS with master...
14:11:19.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:19.567781 3376 ts_manager.cc:194] Registered new tserver with Master: 638f763f19db44b8b70928087de3c6c4 (127.2.121.131:42729)
14:11:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:19.624469 3023 heartbeater.cc:344] Connected to a master server at 127.2.121.190:34499
14:11:19.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:19.627270 3376 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "94af139f04f64b0891ae8e49f5d705fa" instance_seqno: 1777903871343909) as {username='kudu', principal='kudu/127.2.121.129@KRBTEST.COM'} at 127.2.121.129:58359; Asking this server to re-register.
14:11:19.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:19.628712 3023 heartbeater.cc:461] Registering TS with master...
14:11:19.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:19.631932 3376 ts_manager.cc:194] Registered new tserver with Master: 94af139f04f64b0891ae8e49f5d705fa (127.2.121.129:38867)
14:11:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.059191 3429 catalog_manager.cc:1269] Loaded cluster ID: 646f046dc1e24a52a10b732269305767
14:11:20.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.063639 3429 catalog_manager.cc:1562] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: loading cluster ID for follower catalog manager: success
14:11:20.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.092418 3429 catalog_manager.cc:1584] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: acquiring CA information for follower catalog manager: success
14:11:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.100236 3429 catalog_manager.cc:1612] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:20.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.208119 3451 raft_consensus.cc:493] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:11:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.209734 3451 raft_consensus.cc:515] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:20.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.238010 3451 leader_election.cc:290] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783), 3e861f07d8954c4fa14d0eb4457d7f28 (127.2.121.188:44851)
14:11:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:20.276938 3363 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:20.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:20.296343 3363 leader_election.cc:336] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783): Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:20.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:20.301520 3362 leader_election.cc:336] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 3e861f07d8954c4fa14d0eb4457d7f28 (127.2.121.188:44851): Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:20.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.302210 3362 leader_election.cc:304] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a33f80b015d34ff687945c6784be125b; no voters: 3e861f07d8954c4fa14d0eb4457d7f28, 5cbbf08fc96d4a45af7a03eec236c560
14:11:20.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.303867 3451 raft_consensus.cc:2749] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:11:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:20.901403 3439 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1569 milliseconds
14:11:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.903723 3413 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:11:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:19.335888 3440 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:20.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:19.332365 3438 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:20.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.910223 3413 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.920128 3413 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.927907 3413 hybrid_clock.cc:648] HybridClock initialized: now 1777903880927831 us; error 64 us; skew 500 ppm
14:11:20.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.929064 3413 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:20.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:20 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903880, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:20.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.937567 3413 init.cc:377] Logged in from keytab as kudu/127.2.121.188@KRBTEST.COM (short username kudu)
14:11:20.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.958859 3413 webserver.cc:492] Webserver started at http://127.2.121.188:33541/ using document root <none> and password file <none>
14:11:20.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.960431 3413 fs_manager.cc:362] Metadata directory not provided
14:11:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.961930 3413 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:20.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.975607 3413 fs_manager.cc:714] Time spent opening directory manager: real 0.009s user 0.009s sys 0.000s
14:11:20.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.981590 3465 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:20.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.982942 3413 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
14:11:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.983328 3413 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3e861f07d8954c4fa14d0eb4457d7f28"
14:11:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:04 on dist-test-slave-vb5z"
14:11:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:20.986464 3413 fs_report.cc:389] FS layout report
14:11:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data/data
14:11:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:21.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:21 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903881, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.129@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:21.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.042726 2904 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:21.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.053689 3413 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:21.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.060252 3413 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:21.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.060849 3413 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:21.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.155956 3413 rpc_server.cc:307] RPC server started. Bound to: 127.2.121.188:44851
14:11:21.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.156080 3518 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.121.188:44851 every 8 connection(s)
14:11:21.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.159126 3413 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data/info.pb
14:11:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.169221 2534 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 3413
14:11:21.170 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:21.170 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.121.189:43783
14:11:21.171 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:11:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.172752 2534 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data
14:11:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/logs
14:11:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data/info.pb
14:11:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.189:43783
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.189
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36363
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.189
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:11:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.173008 3519 sys_catalog.cc:263] Verifying existing consensus state
14:11:21.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.182163 3519 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: Bootstrap starting.
14:11:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.224706 3519 log.cc:826] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: Log is configured to *not* fsync() on all Append() calls
14:11:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.243122 3519 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:11:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.244158 3519 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: Bootstrap complete.
14:11:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.273751 3519 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.276114 3519 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3e861f07d8954c4fa14d0eb4457d7f28, State: Initialized, Role: FOLLOWER
14:11:21.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.277103 3519 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:21.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.283499 3523 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:21.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.284809 3523 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: This master's current role is: FOLLOWER
14:11:21.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.285498 3519 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: configured and running, proceeding with master startup.
14:11:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.332319 3534 catalog_manager.cc:1269] Loaded cluster ID: 646f046dc1e24a52a10b732269305767
14:11:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.332940 3534 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: loading cluster ID for follower catalog manager: success
14:11:21.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.340907 3534 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: acquiring CA information for follower catalog manager: success
14:11:21.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:21 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903880, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.188@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:21.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.346781 3534 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:21.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.371002 3413 master_runner.cc:186] Error getting master registration for 127.2.121.189:43783: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111),
14:11:21.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.432467 3413 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:11:21.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.583988 3315 heartbeater.cc:344] Connected to a master server at 127.2.121.188:44851
14:11:21.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.584527 3315 heartbeater.cc:499] Master 127.2.121.188:44851 was elected leader, sending a full tablet report...
14:11:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.591315 3484 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "638f763f19db44b8b70928087de3c6c4" instance_seqno: 1777903876359054) as {username='kudu', principal='kudu/127.2.121.131@KRBTEST.COM'} at 127.2.121.131:33145; Asking this server to re-register.
14:11:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.593158 3315 heartbeater.cc:461] Registering TS with master...
14:11:21.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.596841 3483 ts_manager.cc:194] Registered new tserver with Master: 638f763f19db44b8b70928087de3c6c4 (127.2.121.131:42729)
14:11:21.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.657591 3021 heartbeater.cc:344] Connected to a master server at 127.2.121.188:44851
14:11:21.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.660593 3483 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "94af139f04f64b0891ae8e49f5d705fa" instance_seqno: 1777903871343909) as {username='kudu', principal='kudu/127.2.121.129@KRBTEST.COM'} at 127.2.121.129:37425; Asking this server to re-register.
14:11:21.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.662417 3021 heartbeater.cc:461] Registering TS with master...
14:11:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.665844 3483 ts_manager.cc:194] Registered new tserver with Master: 94af139f04f64b0891ae8e49f5d705fa (127.2.121.129:38867)
14:11:21.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:21.708225 3521 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:11:21.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:21.709038 3521 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:21.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:21.710134 3521 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:11:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:21.773883 3521 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:21.774223 3521 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:11:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:21.774706 3521 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:11:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:21.775044 3521 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:21.775388 3521 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:11:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:21.775790 3521 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:21.776041 3521 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34683/jwks.json
14:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.850871 3521 master_runner.cc:386] Master server non-default flags:
14:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data
14:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.189
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.189:43783
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data/info.pb
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.189
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36363
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/logs
14:11:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:21.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:21.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:21.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:21.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:11:21.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:11:21.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:21.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:11:21.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:11:21.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:21.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.852607 3521 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:21.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:21.854244 3521 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:21.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:21.915607 3548 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:22.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.076965 3168 heartbeater.cc:344] Connected to a master server at 127.2.121.188:44851
14:11:22.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.080884 3483 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "9f94afdf18ee48769d301f9bc00ff0c9" instance_seqno: 1777903873751575) as {username='kudu', principal='kudu/127.2.121.130@KRBTEST.COM'} at 127.2.121.130:53499; Asking this server to re-register.
14:11:22.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.082929 3168 heartbeater.cc:461] Registering TS with master...
14:11:22.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.086665 3483 ts_manager.cc:194] Registered new tserver with Master: 9f94afdf18ee48769d301f9bc00ff0c9 (127.2.121.130:34419)
14:11:22.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.355938 3555 raft_consensus.cc:493] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:11:22.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.357975 3555 raft_consensus.cc:515] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:22.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.383791 3555 leader_election.cc:290] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783), 3e861f07d8954c4fa14d0eb4457d7f28 (127.2.121.188:44851)
14:11:22.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:22.395761 3363 leader_election.cc:336] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783): Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.451262 3494 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a33f80b015d34ff687945c6784be125b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" is_pre_election: true
14:11:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.452648 3494 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a33f80b015d34ff687945c6784be125b in term 1.
14:11:22.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.455169 3362 leader_election.cc:304] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3e861f07d8954c4fa14d0eb4457d7f28, a33f80b015d34ff687945c6784be125b; no voters: 5cbbf08fc96d4a45af7a03eec236c560
14:11:22.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.456842 3555 raft_consensus.cc:2804] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 1 FOLLOWER]: Leader pre-election won for term 2
14:11:22.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.457331 3555 raft_consensus.cc:493] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:11:22.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.457764 3555 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 1 FOLLOWER]: Advancing to term 2
14:11:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.487048 3555 raft_consensus.cc:515] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:22.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.490412 3494 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a33f80b015d34ff687945c6784be125b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "3e861f07d8954c4fa14d0eb4457d7f28"
14:11:22.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.490967 3494 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 1 FOLLOWER]: Advancing to term 2
14:11:22.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:22.493603 3363 leader_election.cc:336] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783): Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:22.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.523617 3555 leader_election.cc:290] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 2 election: Requested vote from peers 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783), 3e861f07d8954c4fa14d0eb4457d7f28 (127.2.121.188:44851)
14:11:22.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.525523 3494 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a33f80b015d34ff687945c6784be125b in term 2.
14:11:22.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.527840 3362 leader_election.cc:304] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3e861f07d8954c4fa14d0eb4457d7f28, a33f80b015d34ff687945c6784be125b; no voters: 5cbbf08fc96d4a45af7a03eec236c560
14:11:22.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.529798 3555 raft_consensus.cc:2804] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 2 FOLLOWER]: Leader election won for term 2
14:11:22.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.540825 3555 raft_consensus.cc:697] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 2 LEADER]: Becoming Leader. State: Replica: a33f80b015d34ff687945c6784be125b, State: Running, Role: LEADER
14:11:22.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.542765 3555 consensus_queue.cc:237] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:22.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.573230 3560 sys_catalog.cc:455] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: SysCatalogTable state changed. Reason: New leader a33f80b015d34ff687945c6784be125b. Latest consensus state: current_term: 2 leader_uuid: "a33f80b015d34ff687945c6784be125b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:22.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.575006 3560 sys_catalog.cc:458] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: This master's current role is: LEADER
14:11:22.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.588279 3563 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
14:11:22.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.627743 3563 catalog_manager.cc:1494] Initializing Kudu cluster ID...
14:11:22.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.636178 3563 catalog_manager.cc:1269] Loaded cluster ID: 646f046dc1e24a52a10b732269305767
14:11:22.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.637051 3563 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
14:11:22.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.650071 3563 catalog_manager.cc:1514] Loading token signing keys...
14:11:22.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.672003 3563 catalog_manager.cc:6055] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: Loaded TSK: 0
14:11:22.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:22.674549 3563 catalog_manager.cc:1524] Initializing in-progress tserver states...
14:11:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:22.937842 3363 consensus_peers.cc:597] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b -> Peer 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783): Couldn't send request to peer 5cbbf08fc96d4a45af7a03eec236c560. Status: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:11:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.063714 3494 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 2 FOLLOWER]: Refusing update from remote peer a33f80b015d34ff687945c6784be125b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
14:11:23.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.066468 3560 consensus_queue.cc:1048] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
14:11:23.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.114135 3567 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a33f80b015d34ff687945c6784be125b. Latest consensus state: current_term: 2 leader_uuid: "a33f80b015d34ff687945c6784be125b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:23.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.115876 3567 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: This master's current role is: FOLLOWER
14:11:23.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.128057 3560 sys_catalog.cc:455] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "a33f80b015d34ff687945c6784be125b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:23.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.128794 3560 sys_catalog.cc:458] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: This master's current role is: LEADER
14:11:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.137951 3567 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "a33f80b015d34ff687945c6784be125b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.139539 3567 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: This master's current role is: FOLLOWER
14:11:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:23 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903883, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.130@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:23.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.488549 3051 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:23.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:23.295868 3547 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 3521
14:11:23.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:23.872073 3547 kernel_stack_watchdog.cc:198] Thread 3521 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 378ms:
14:11:23.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:11:23.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:11:23.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:11:23.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:11:23.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:21.917766 3549 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:23.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:23.877722 3521 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.963s user 0.703s sys 1.078s
14:11:23.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:23.891754 3521 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.977s user 0.703s sys 1.078s
14:11:23.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:23.902758 3577 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:23.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.906569 3521 server_base.cc:1061] running on GCE node
14:11:23.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.907956 3521 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.925441 3521 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.933145 3521 hybrid_clock.cc:648] HybridClock initialized: now 1777903883933084 us; error 64 us; skew 500 ppm
14:11:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.934330 3521 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:23.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:23 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903883, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:23.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.951372 3521 init.cc:377] Logged in from keytab as kudu/127.2.121.189@KRBTEST.COM (short username kudu)
14:11:23.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.972934 3521 webserver.cc:492] Webserver started at http://127.2.121.189:36363/ using document root <none> and password file <none>
14:11:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.974231 3521 fs_manager.cc:362] Metadata directory not provided
14:11:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.974555 3521 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:23.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.986513 3521 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.005s sys 0.005s
14:11:23.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.992545 3583 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:23.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.994311 3521 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
14:11:23.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.994729 3521 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:23.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5cbbf08fc96d4a45af7a03eec236c560"
14:11:23.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:01 on dist-test-slave-vb5z"
14:11:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:23.997510 3521 fs_report.cc:389] FS layout report
14:11:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data/data
14:11:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:24.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.084388 3170 heartbeater.cc:499] Master 127.2.121.190:34499 was elected leader, sending a full tablet report...
14:11:24.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.093575 3330 master_runner.cc:186] Error getting master registration for 127.2.121.189:43783: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111),
14:11:24.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.093953 3330 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:11:24.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.097154 3521 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.107669 3521 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.108270 3521 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:24.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.214728 3521 rpc_server.cc:307] RPC server started. Bound to: 127.2.121.189:43783
14:11:24.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.214799 3637 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.121.189:43783 every 8 connection(s)
14:11:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.217846 3521 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data/info.pb
14:11:24.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.226066 2534 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 3521
14:11:24.227 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:24.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.230839 3638 sys_catalog.cc:263] Verifying existing consensus state
14:11:24.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.239259 3638 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: Bootstrap starting.
14:11:24.279 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.121.190:34499(127.2.121.190:34499)] connecting to peer
14:11:24.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.299921 3638 log.cc:826] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: Log is configured to *not* fsync() on all Append() calls
14:11:24.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.321709 3638 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:11:24.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.322754 3638 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: Bootstrap complete.
14:11:24.347 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.121.188:44851(127.2.121.188:44851)] connecting to peer
14:11:24.357 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.121.189:43783(127.2.121.189:43783)] connecting to peer
14:11:24.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.357228 3638 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:24.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.360666 3638 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5cbbf08fc96d4a45af7a03eec236c560, State: Initialized, Role: FOLLOWER
14:11:24.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.361917 3638 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.371735 3646 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.372637 3646 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: This master's current role is: FOLLOWER
14:11:24.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.376561 3638 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: configured and running, proceeding with master startup.
14:11:24.406 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.2.121.188:44851(127.2.121.188:44851)] Successfully connected to peer
14:11:24.416 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.2.121.189:43783(127.2.121.189:43783)] Successfully connected to peer
14:11:24.433 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.2.121.190:34499(127.2.121.190:34499)] Successfully connected to peer
14:11:24.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:24 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903878, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.190@KRBTEST.COM for kudu/127.2.121.189@KRBTEST.COM
14:11:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.560985 3662 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:11:24.573 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
14:11:24.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.584064 3613 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [term 1 FOLLOWER]: Advancing to term 2
14:11:24.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.624913 3317 heartbeater.cc:499] Master 127.2.121.190:34499 was elected leader, sending a full tablet report...
14:11:24.627 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.121.188
14:11:24.630 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.121.190
14:11:24.657 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.121.189
14:11:24.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:24 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903883, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.189@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.709959 3646 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a33f80b015d34ff687945c6784be125b. Latest consensus state: current_term: 2 leader_uuid: "a33f80b015d34ff687945c6784be125b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:24.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.710676 3646 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: This master's current role is: FOLLOWER
14:11:24.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.712119 3668 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "a33f80b015d34ff687945c6784be125b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.712878 3668 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: This master's current role is: FOLLOWER
14:11:24.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:24 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903883, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.189@KRBTEST.COM for kudu/127.2.121.189@KRBTEST.COM
14:11:24.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:24.730482 1884 debug-util.cc:398] Leaking SignalData structure 0x7b080009fb60 after lost signal to thread 1821
14:11:24.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.726457 3316 heartbeater.cc:344] Connected to a master server at 127.2.121.189:43783
14:11:24.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:24.731237 1884 debug-util.cc:398] Leaking SignalData structure 0x7b0800074f00 after lost signal to thread 1887
14:11:24.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.736014 3023 heartbeater.cc:499] Master 127.2.121.190:34499 was elected leader, sending a full tablet report...
14:11:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.737183 3603 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "638f763f19db44b8b70928087de3c6c4" instance_seqno: 1777903876359054) as {username='kudu', principal='kudu/127.2.121.131@KRBTEST.COM'} at 127.2.121.131:39553; Asking this server to re-register.
14:11:24.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.739215 3316 heartbeater.cc:461] Registering TS with master...
14:11:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.743569 3602 ts_manager.cc:194] Registered new tserver with Master: 638f763f19db44b8b70928087de3c6c4 (127.2.121.131:42729)
14:11:24.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:24 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903883, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.189@KRBTEST.COM for kudu/127.2.121.188@KRBTEST.COM
14:11:24.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:24 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903881, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.129@KRBTEST.COM for kudu/127.2.121.189@KRBTEST.COM
14:11:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.814143 3022 heartbeater.cc:344] Connected to a master server at 127.2.121.189:43783
14:11:24.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.817247 3603 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "94af139f04f64b0891ae8e49f5d705fa" instance_seqno: 1777903871343909) as {username='kudu', principal='kudu/127.2.121.129@KRBTEST.COM'} at 127.2.121.129:46775; Asking this server to re-register.
14:11:24.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.818586 3022 heartbeater.cc:461] Registering TS with master...
14:11:24.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.823339 3603 ts_manager.cc:194] Registered new tserver with Master: 94af139f04f64b0891ae8e49f5d705fa (127.2.121.129:38867)
14:11:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.984988 3674 sys_catalog.cc:455] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "a33f80b015d34ff687945c6784be125b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:24.987573 3674 sys_catalog.cc:458] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: This master's current role is: LEADER
14:11:25.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:25 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903883, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.130@KRBTEST.COM for kudu/127.2.121.189@KRBTEST.COM
14:11:25.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:25.126134 3169 heartbeater.cc:344] Connected to a master server at 127.2.121.189:43783
14:11:25.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:25.130051 3603 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "9f94afdf18ee48769d301f9bc00ff0c9" instance_seqno: 1777903873751575) as {username='kudu', principal='kudu/127.2.121.130@KRBTEST.COM'} at 127.2.121.130:42721; Asking this server to re-register.
14:11:25.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:25.131930 3169 heartbeater.cc:461] Registering TS with master...
14:11:25.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:25.135617 3603 ts_manager.cc:194] Registered new tserver with Master: 9f94afdf18ee48769d301f9bc00ff0c9 (127.2.121.130:34419)
14:11:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:25 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1777903876, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.121.189@KRBTEST.COM
14:11:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:25 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1777903876, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.121.188@KRBTEST.COM
14:11:25.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:25 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1777903876, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:25.346 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xbe6978ec, L:/127.0.0.1:58614 - R:/127.2.121.189:43783] using SASL/GSSAPI
14:11:25.352 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x446e42fc, L:/127.0.0.1:39234 - R:/127.2.121.190:34499] using SASL/GSSAPI
14:11:25.379 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xcba5b19a, L:/127.0.0.1:36454 - R:/127.2.121.188:44851] using SASL/GSSAPI
14:11:25.436 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:11:25.438 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1207), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.121.190:34499, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.121.188:44851, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.121.189:43783, [1152ms] Sub RPC ConnectToMaster: received response from server master-127.2.121.189:43783: OK, [1156ms] Sub RPC ConnectToMaster: received response from server master-127.2.121.188:44851: OK, [1156ms] Sub RPC ConnectToMaster: received response from server master-127.2.121.190:34499: OK, deferred=null) after lookup
14:11:25.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:25.578413 3662 catalog_manager.cc:1269] Loaded cluster ID: 646f046dc1e24a52a10b732269305767
14:11:25.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:25.578727 3662 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: loading cluster ID for follower catalog manager: success
14:11:25.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:25.587870 3662 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: acquiring CA information for follower catalog manager: success
14:11:25.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:25.595003 3662 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:25.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:25.637490 3407 debug-util.cc:398] Leaking SignalData structure 0x7b0800071520 after lost signal to thread 3332
14:11:25.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:25.638273 3407 debug-util.cc:398] Leaking SignalData structure 0x7b0800094cc0 after lost signal to thread 3410
14:11:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:26 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903886, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.131@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:26.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:26.049531 3198 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:26.477 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:11:26.478 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:11:26.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:26 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903886, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:26.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:11:26.572 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:26.573 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc
14:11:26.575 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:11:26.575 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.121.190:34499
14:11:26.581 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:11:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:26.582477 2534 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 3330
14:11:26.633 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.2.121.190:34499(127.2.121.190:34499)] handling channelInactive
14:11:26.634 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.121.190:34499(127.2.121.190:34499)] cleaning up while in state READY due to: connection closed
14:11:26.635 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:26.635 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.121.188:44851
14:11:26.635 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:11:26.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:26.636261 2534 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 3413
14:11:26.674 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.2.121.188:44851(127.2.121.188:44851)] handling channelInactive
14:11:26.674 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.121.188:44851(127.2.121.188:44851)] cleaning up while in state READY due to: connection closed
14:11:26.677 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:26.677 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.121.189:43783
14:11:26.677 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:11:26.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:26.678270 2534 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 3521
14:11:26.711 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.2.121.189:43783(127.2.121.189:43783)] handling channelInactive
14:11:26.711 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.2.121.189:43783(127.2.121.189:43783)] cleaning up while in state READY due to: connection closed
14:11:26.712 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:26.712 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.121.190:34499
14:11:26.712 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:11:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:26.714187 2534 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/logs
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data/info.pb
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.190:34499
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.190
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38001
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.190
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:11:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:27.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:27.154986 3687 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:11:27.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:27.155616 3687 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:27.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:27.156239 3687 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:11:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:27.190385 3687 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:27.190644 3687 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:11:27.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:27.190917 3687 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:11:27.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:27.191172 3687 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:27.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:27.191571 3687 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:11:27.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:27.191908 3687 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:27.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:27.192162 3687 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34683/jwks.json
14:11:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:27.234551 3687 master_runner.cc:386] Master server non-default flags:
14:11:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data
14:11:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:11:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.190
14:11:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.190:34499
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data/info.pb
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.190
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38001
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/logs
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:11:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:11:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:11:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:27.236548 3687 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:27.238010 3687 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:27.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:27.251875 3694 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:28.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:27.254251 3695 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:28.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:28.618193 3687 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.367s user 0.546s sys 0.814s
14:11:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:28.618656 3687 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.368s user 0.546s sys 0.814s
14:11:28.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.623054 3687 server_base.cc:1061] running on GCE node
14:11:28.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:28.623348 3707 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:28.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.624701 3687 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.627580 3687 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:28.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.628993 3687 hybrid_clock.cc:648] HybridClock initialized: now 1777903888628955 us; error 50 us; skew 500 ppm
14:11:28.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.629954 3687 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:28.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:28 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903888, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:28.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.638536 3687 init.cc:377] Logged in from keytab as kudu/127.2.121.190@KRBTEST.COM (short username kudu)
14:11:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.645087 3687 webserver.cc:492] Webserver started at http://127.2.121.190:38001/ using document root <none> and password file <none>
14:11:28.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.646175 3687 fs_manager.cc:362] Metadata directory not provided
14:11:28.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.646481 3687 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:28.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.656963 3687 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.007s sys 0.001s
14:11:28.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.661890 3714 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:28.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.663300 3687 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
14:11:28.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.663791 3687 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:11:28.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a33f80b015d34ff687945c6784be125b"
14:11:28.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:59 on dist-test-slave-vb5z"
14:11:28.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.665885 3687 fs_report.cc:389] FS layout report
14:11:28.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:28.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:11:28.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:11:28.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data/data
14:11:28.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:28.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:28.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:28.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:28.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:28.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:28.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:28.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:28.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:28.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:28.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:28.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.757668 3687 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:28.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.764273 3687 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:28.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.764722 3687 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:28.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.847759 3687 rpc_server.cc:307] RPC server started. Bound to: 127.2.121.190:34499
14:11:28.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.847821 3766 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.121.190:34499 every 8 connection(s)
14:11:28.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.850484 3687 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data/info.pb
14:11:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.854568 2534 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 3687
14:11:28.855 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:28.855 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.121.188:44851
14:11:28.855 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:11:28.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.857489 2534 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/logs
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data/info.pb
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.188:44851
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.188
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33541
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.188
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:28.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:28.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:11:28.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:28.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.865728 3767 sys_catalog.cc:263] Verifying existing consensus state
14:11:28.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.871431 3767 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: Bootstrap starting.
14:11:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.917953 3767 log.cc:826] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: Log is configured to *not* fsync() on all Append() calls
14:11:28.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.941470 3767 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:11:28.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.943009 3767 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: Bootstrap complete.
14:11:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.980907 3767 raft_consensus.cc:359] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.984870 3767 raft_consensus.cc:740] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: a33f80b015d34ff687945c6784be125b, State: Initialized, Role: FOLLOWER
14:11:28.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.986016 3767 consensus_queue.cc:260] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:28.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.989096 3771 sys_catalog.cc:455] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:28.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.989755 3771 sys_catalog.cc:458] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: This master's current role is: FOLLOWER
14:11:28.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:28.991339 3767 sys_catalog.cc:565] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: configured and running, proceeding with master startup.
14:11:29.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.033190 3782 catalog_manager.cc:1269] Loaded cluster ID: 646f046dc1e24a52a10b732269305767
14:11:29.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.033842 3782 catalog_manager.cc:1562] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: loading cluster ID for follower catalog manager: success
14:11:29.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.042126 3782 catalog_manager.cc:1584] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: acquiring CA information for follower catalog manager: success
14:11:29.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:29 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903888, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.190@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.048236 3782 catalog_manager.cc:1612] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:29.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.064736 3687 master_runner.cc:186] Error getting master registration for 127.2.121.189:43783: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111),
14:11:29.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.084494 3687 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:11:29.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.119061 3170 heartbeater.cc:344] Connected to a master server at 127.2.121.190:34499
14:11:29.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.123713 3732 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "9f94afdf18ee48769d301f9bc00ff0c9" instance_seqno: 1777903873751575) as {username='kudu', principal='kudu/127.2.121.130@KRBTEST.COM'} at 127.2.121.130:38493; Asking this server to re-register.
14:11:29.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.125151 3170 heartbeater.cc:461] Registering TS with master...
14:11:29.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.128410 3732 ts_manager.cc:194] Registered new tserver with Master: 9f94afdf18ee48769d301f9bc00ff0c9 (127.2.121.130:34419)
14:11:29.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:29.289340 3769 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:11:29.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:29.289893 3769 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:29.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:29.290596 3769 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:11:29.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:29.327868 3769 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:29.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:29.328111 3769 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:11:29.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:29.328408 3769 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:11:29.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:29.328733 3769 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:29.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:29.329053 3769 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:11:29.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:29.329352 3769 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:29.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:29.329543 3769 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34683/jwks.json
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.371585 3769 master_runner.cc:386] Master server non-default flags:
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.188
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.188:44851
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data/info.pb
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.188
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33541
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/logs
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.373028 3769 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.374425 3769 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:29.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:29.387753 3792 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:29.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.731599 3317 heartbeater.cc:344] Connected to a master server at 127.2.121.190:34499
14:11:29.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.735674 3732 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "638f763f19db44b8b70928087de3c6c4" instance_seqno: 1777903876359054) as {username='kudu', principal='kudu/127.2.121.131@KRBTEST.COM'} at 127.2.121.131:37467; Asking this server to re-register.
14:11:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.739065 3317 heartbeater.cc:461] Registering TS with master...
14:11:29.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.744349 3732 ts_manager.cc:194] Registered new tserver with Master: 638f763f19db44b8b70928087de3c6c4 (127.2.121.131:42729)
14:11:29.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.775044 3023 heartbeater.cc:344] Connected to a master server at 127.2.121.190:34499
14:11:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.780005 3732 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "94af139f04f64b0891ae8e49f5d705fa" instance_seqno: 1777903871343909) as {username='kudu', principal='kudu/127.2.121.129@KRBTEST.COM'} at 127.2.121.129:56305; Asking this server to re-register.
14:11:29.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.784284 3023 heartbeater.cc:461] Registering TS with master...
14:11:29.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:29.789726 3732 ts_manager.cc:194] Registered new tserver with Master: 94af139f04f64b0891ae8e49f5d705fa (127.2.121.129:38867)
14:11:30.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:30.423614 1994 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.1.196.62:34245, user_credentials={real_user=slave}} blocked reactor thread for 46377.5us
14:11:30.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:30.430994 3807 raft_consensus.cc:493] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:11:30.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:30.436046 3807 raft_consensus.cc:515] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:30.470216 3807 leader_election.cc:290] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783), 3e861f07d8954c4fa14d0eb4457d7f28 (127.2.121.188:44851)
14:11:30.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:30.486941 3719 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:30.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:30.509084 3718 leader_election.cc:336] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3e861f07d8954c4fa14d0eb4457d7f28 (127.2.121.188:44851): Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:30.513309 3719 leader_election.cc:336] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783): Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:30.514351 3719 leader_election.cc:304] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a33f80b015d34ff687945c6784be125b; no voters: 3e861f07d8954c4fa14d0eb4457d7f28, 5cbbf08fc96d4a45af7a03eec236c560
14:11:30.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:30.515806 3807 raft_consensus.cc:2749] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
14:11:31.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:29.392761 3793 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:31.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:29.400346 3795 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:31.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:31.014258 3794 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1624 milliseconds
14:11:31.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.014389 3769 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:11:31.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.016521 3769 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:31.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.031414 3769 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:31.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.035146 3769 hybrid_clock.cc:648] HybridClock initialized: now 1777903891034899 us; error 230 us; skew 500 ppm
14:11:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.036609 3769 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:31 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903891, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.129@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.046327 2904 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:31 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903891, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:31.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.049101 3769 init.cc:377] Logged in from keytab as kudu/127.2.121.188@KRBTEST.COM (short username kudu)
14:11:31.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.058454 3769 webserver.cc:492] Webserver started at http://127.2.121.188:33541/ using document root <none> and password file <none>
14:11:31.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.060117 3769 fs_manager.cc:362] Metadata directory not provided
14:11:31.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.060513 3769 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:31.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.073982 3769 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.001s sys 0.008s
14:11:31.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.079197 3820 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:31.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.080591 3769 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
14:11:31.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.080904 3769 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3e861f07d8954c4fa14d0eb4457d7f28"
14:11:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:04 on dist-test-slave-vb5z"
14:11:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.082850 3769 fs_report.cc:389] FS layout report
14:11:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data/data
14:11:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:31.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:31.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:31.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:31.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:31.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:31.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.178799 3769 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:31.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.186851 3769 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:31.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.187424 3769 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:31.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.296610 3769 rpc_server.cc:307] RPC server started. Bound to: 127.2.121.188:44851
14:11:31.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.296645 3873 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.121.188:44851 every 8 connection(s)
14:11:31.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.300654 3769 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data/info.pb
14:11:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.303193 2534 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 3769
14:11:31.304 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:31.304 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.121.189:43783
14:11:31.304 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:11:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.307026 2534 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data
14:11:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/logs
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data/info.pb
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.189:43783
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.189
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36363
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.189
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:31.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:11:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:31.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.316855 3874 sys_catalog.cc:263] Verifying existing consensus state
14:11:31.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.330552 3874 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: Bootstrap starting.
14:11:31.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.389966 3874 log.cc:826] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: Log is configured to *not* fsync() on all Append() calls
14:11:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.429227 3874 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:11:31.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.430316 3874 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: Bootstrap complete.
14:11:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.468506 3874 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.472150 3874 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3e861f07d8954c4fa14d0eb4457d7f28, State: Initialized, Role: FOLLOWER
14:11:31.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.473208 3874 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:31.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.476322 3878 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:31.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.477495 3878 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: This master's current role is: FOLLOWER
14:11:31.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.478201 3874 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: configured and running, proceeding with master startup.
14:11:31.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.541756 3889 catalog_manager.cc:1269] Loaded cluster ID: 646f046dc1e24a52a10b732269305767
14:11:31.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.542413 3889 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: loading cluster ID for follower catalog manager: success
14:11:31.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:31 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903891, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.188@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:31.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.588359 3889 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: acquiring CA information for follower catalog manager: success
14:11:31.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.611389 3889 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:31.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.608872 3769 master_runner.cc:186] Error getting master registration for 127.2.121.189:43783: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111),
14:11:31.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.637255 3769 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:11:31.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.707954 3315 heartbeater.cc:344] Connected to a master server at 127.2.121.188:44851
14:11:31.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.715734 3839 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "638f763f19db44b8b70928087de3c6c4" instance_seqno: 1777903876359054) as {username='kudu', principal='kudu/127.2.121.131@KRBTEST.COM'} at 127.2.121.131:43715; Asking this server to re-register.
14:11:31.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.718106 3315 heartbeater.cc:461] Registering TS with master...
14:11:31.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.723307 3838 ts_manager.cc:194] Registered new tserver with Master: 638f763f19db44b8b70928087de3c6c4 (127.2.121.131:42729)
14:11:31.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.761021 3021 heartbeater.cc:344] Connected to a master server at 127.2.121.188:44851
14:11:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.764067 3839 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "94af139f04f64b0891ae8e49f5d705fa" instance_seqno: 1777903871343909) as {username='kudu', principal='kudu/127.2.121.129@KRBTEST.COM'} at 127.2.121.129:44741; Asking this server to re-register.
14:11:31.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.765920 3021 heartbeater.cc:461] Registering TS with master...
14:11:31.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.772166 3839 ts_manager.cc:194] Registered new tserver with Master: 94af139f04f64b0891ae8e49f5d705fa (127.2.121.129:38867)
14:11:31.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:31.836493 3876 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:11:31.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:31.837589 3876 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:31.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:31.838850 3876 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:11:31.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:31.900266 3876 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:31.901810 3876 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:11:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:31.902511 3876 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:11:31.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:31.903072 3876 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:31.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:31.903694 3876 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:11:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:31.904173 3876 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:31.904582 3876 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34683/jwks.json
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.973619 3876 master_runner.cc:386] Master server non-default flags:
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.189
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.189:43783
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data/info.pb
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.189
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36363
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/logs
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:11:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:31.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.977279 3876 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:31.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:31.979396 3876 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:32.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:32.025255 3902 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:32.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.069767 3906 raft_consensus.cc:493] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:11:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.071056 3906 raft_consensus.cc:515] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:32.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.098059 3906 leader_election.cc:290] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783), 3e861f07d8954c4fa14d0eb4457d7f28 (127.2.121.188:44851)
14:11:32.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:32.126418 3719 leader_election.cc:336] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783): Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:32.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.132416 3849 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a33f80b015d34ff687945c6784be125b" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" is_pre_election: true
14:11:32.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.133630 3849 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a33f80b015d34ff687945c6784be125b in term 2.
14:11:32.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.136042 3718 leader_election.cc:304] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3e861f07d8954c4fa14d0eb4457d7f28, a33f80b015d34ff687945c6784be125b; no voters: 5cbbf08fc96d4a45af7a03eec236c560
14:11:32.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.137202 3906 raft_consensus.cc:2804] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 2 FOLLOWER]: Leader pre-election won for term 3
14:11:32.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.137538 3906 raft_consensus.cc:493] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:11:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.137843 3906 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 2 FOLLOWER]: Advancing to term 3
14:11:32.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.148537 3906 raft_consensus.cc:515] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:32.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.161278 3906 leader_election.cc:290] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 3 election: Requested vote from peers 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783), 3e861f07d8954c4fa14d0eb4457d7f28 (127.2.121.188:44851)
14:11:32.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.162479 3849 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a33f80b015d34ff687945c6784be125b" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "3e861f07d8954c4fa14d0eb4457d7f28"
14:11:32.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.163158 3849 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 2 FOLLOWER]: Advancing to term 3
14:11:32.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:32.173022 3719 leader_election.cc:336] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783): Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.202787 3849 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a33f80b015d34ff687945c6784be125b in term 3.
14:11:32.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.204720 3718 leader_election.cc:304] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3e861f07d8954c4fa14d0eb4457d7f28, a33f80b015d34ff687945c6784be125b; no voters: 5cbbf08fc96d4a45af7a03eec236c560
14:11:32.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.209233 3906 raft_consensus.cc:2804] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 3 FOLLOWER]: Leader election won for term 3
14:11:32.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.214694 3168 heartbeater.cc:344] Connected to a master server at 127.2.121.188:44851
14:11:32.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.217303 3839 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "9f94afdf18ee48769d301f9bc00ff0c9" instance_seqno: 1777903873751575) as {username='kudu', principal='kudu/127.2.121.130@KRBTEST.COM'} at 127.2.121.130:33965; Asking this server to re-register.
14:11:32.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.220562 3168 heartbeater.cc:461] Registering TS with master...
14:11:32.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.224792 3839 ts_manager.cc:194] Registered new tserver with Master: 9f94afdf18ee48769d301f9bc00ff0c9 (127.2.121.130:34419)
14:11:32.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.234093 3906 raft_consensus.cc:697] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 3 LEADER]: Becoming Leader. State: Replica: a33f80b015d34ff687945c6784be125b, State: Running, Role: LEADER
14:11:32.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.235948 3906 consensus_queue.cc:237] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:32.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.251775 3913 sys_catalog.cc:455] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: SysCatalogTable state changed. Reason: New leader a33f80b015d34ff687945c6784be125b. Latest consensus state: current_term: 3 leader_uuid: "a33f80b015d34ff687945c6784be125b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:32.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.253068 3913 sys_catalog.cc:458] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: This master's current role is: LEADER
14:11:32.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.264248 3915 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
14:11:32.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.304352 3915 catalog_manager.cc:1494] Initializing Kudu cluster ID...
14:11:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.309171 3915 catalog_manager.cc:1269] Loaded cluster ID: 646f046dc1e24a52a10b732269305767
14:11:32.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.320326 3915 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
14:11:32.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.325914 3915 catalog_manager.cc:1514] Loading token signing keys...
14:11:32.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.341686 3915 catalog_manager.cc:6055] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: Loaded TSK: 0
14:11:32.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.348021 3915 catalog_manager.cc:1524] Initializing in-progress tserver states...
14:11:32.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:32.623718 3719 consensus_peers.cc:597] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b -> Peer 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783): Couldn't send request to peer 5cbbf08fc96d4a45af7a03eec236c560. Status: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:11:32.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.690542 3849 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 3 FOLLOWER]: Refusing update from remote peer a33f80b015d34ff687945c6784be125b: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
14:11:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.693574 3913 consensus_queue.cc:1048] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
14:11:32.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.720554 3913 sys_catalog.cc:455] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "a33f80b015d34ff687945c6784be125b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:32.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.721305 3913 sys_catalog.cc:458] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: This master's current role is: LEADER
14:11:32.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.728715 3921 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "a33f80b015d34ff687945c6784be125b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:32.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.735709 3921 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: This master's current role is: FOLLOWER
14:11:32.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.737500 3917 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a33f80b015d34ff687945c6784be125b. Latest consensus state: current_term: 3 leader_uuid: "a33f80b015d34ff687945c6784be125b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:32.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:32.738690 3917 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: This master's current role is: FOLLOWER
14:11:33.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:33 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903893, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.130@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:33.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:33.492578 3051 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:33.511219 3763 debug-util.cc:398] Leaking SignalData structure 0x7b0800071340 after lost signal to thread 3689
14:11:33.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:33.512173 3763 debug-util.cc:398] Leaking SignalData structure 0x7b0800071180 after lost signal to thread 3766
14:11:33.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:33.674996 1959 debug-util.cc:398] Leaking SignalData structure 0x7b0800079b40 after lost signal to thread 1894
14:11:33.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:33.675817 1959 debug-util.cc:398] Leaking SignalData structure 0x7b0800098100 after lost signal to thread 1962
14:11:33.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:33.758706 3317 heartbeater.cc:499] Master 127.2.121.190:34499 was elected leader, sending a full tablet report...
14:11:33.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:33.803793 3023 heartbeater.cc:499] Master 127.2.121.190:34499 was elected leader, sending a full tablet report...
14:11:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:33.409778 3901 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 3876
14:11:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:34.144639 3901 kernel_stack_watchdog.cc:198] Thread 3876 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 376ms:
14:11:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:11:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:11:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:11:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:11:34.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:32.040724 3904 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:34.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:34.145975 3876 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.114s user 0.624s sys 1.251s
14:11:34.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:34.146308 3876 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.114s user 0.624s sys 1.251s
14:11:34.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:34.159192 3934 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:34.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.163163 3170 heartbeater.cc:499] Master 127.2.121.190:34499 was elected leader, sending a full tablet report...
14:11:34.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.167665 3876 server_base.cc:1061] running on GCE node
14:11:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.169401 3876 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.177958 3687 master_runner.cc:186] Error getting master registration for 127.2.121.189:43783: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111),
14:11:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.178390 3687 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:11:34.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.180001 3876 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.187759 3876 hybrid_clock.cc:648] HybridClock initialized: now 1777903894187694 us; error 72 us; skew 500 ppm
14:11:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.189002 3876 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:34.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:34 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903894, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.203380 3876 init.cc:377] Logged in from keytab as kudu/127.2.121.189@KRBTEST.COM (short username kudu)
14:11:34.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.255358 3876 webserver.cc:492] Webserver started at http://127.2.121.189:36363/ using document root <none> and password file <none>
14:11:34.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.256968 3876 fs_manager.cc:362] Metadata directory not provided
14:11:34.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.257408 3876 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:34.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.309891 3876 fs_manager.cc:714] Time spent opening directory manager: real 0.044s user 0.005s sys 0.001s
14:11:34.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.339176 3942 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:34.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.341004 3876 fs_manager.cc:730] Time spent opening block manager: real 0.027s user 0.008s sys 0.003s
14:11:34.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.341601 3876 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:34.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5cbbf08fc96d4a45af7a03eec236c560"
14:11:34.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:01 on dist-test-slave-vb5z"
14:11:34.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.344661 3876 fs_report.cc:389] FS layout report
14:11:34.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:34.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:34.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:34.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data/data
14:11:34.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:34.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:34.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:34.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:34.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:34.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:34.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:34.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:34.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:34.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:34.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:34.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.456459 3876 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:34.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.465651 3876 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:34.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.466241 3876 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.858551 3996 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.121.189:43783 every 8 connection(s)
14:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.859153 3876 rpc_server.cc:307] RPC server started. Bound to: 127.2.121.189:43783
14:11:34.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.862982 3876 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data/info.pb
14:11:34.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.870415 2534 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 3876
14:11:34.871 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:34.872 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.121.190:34499(127.2.121.190:34499)] connecting to peer
14:11:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.906412 3997 sys_catalog.cc:263] Verifying existing consensus state
14:11:34.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.916010 3997 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: Bootstrap starting.
14:11:34.924 [DEBUG - kudu-nio-3] (Connection.java:757) [peer master-127.2.121.190:34499(127.2.121.190:34499)] Successfully connected to peer
14:11:34.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:34 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903891, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.129@KRBTEST.COM for kudu/127.2.121.189@KRBTEST.COM
14:11:34.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.960314 3022 heartbeater.cc:344] Connected to a master server at 127.2.121.189:43783
14:11:34.985 [DEBUG - kudu-nio-3] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.121.190
14:11:34.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:34.991969 3997 log.cc:826] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: Log is configured to *not* fsync() on all Append() calls
14:11:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.088119 3997 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:11:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.089205 3997 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: Bootstrap complete.
14:11:35.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:35 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903888, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.190@KRBTEST.COM for kudu/127.2.121.189@KRBTEST.COM
14:11:35.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.121402 3997 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.124854 3997 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5cbbf08fc96d4a45af7a03eec236c560, State: Initialized, Role: FOLLOWER
14:11:35.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.126019 3997 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:35.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.162483 3971 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [term 2 FOLLOWER]: Advancing to term 3
14:11:35.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:35 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1777903886, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:35.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.161356 4004 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:35.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.174620 4004 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: This master's current role is: FOLLOWER
14:11:35.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.175297 3997 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: configured and running, proceeding with master startup.
14:11:35.249 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x1c2b17d3, L:/127.0.0.1:39330 - R:/127.2.121.190:34499] using SASL/GSSAPI
14:11:35.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.262884 4005 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "a33f80b015d34ff687945c6784be125b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:35.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.263839 4005 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: This master's current role is: FOLLOWER
14:11:35.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.275039 4004 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a33f80b015d34ff687945c6784be125b. Latest consensus state: current_term: 3 leader_uuid: "a33f80b015d34ff687945c6784be125b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:35.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.276510 4004 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: This master's current role is: FOLLOWER
14:11:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.299824 4020 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:11:35.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:35 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903893, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.130@KRBTEST.COM for kudu/127.2.121.189@KRBTEST.COM
14:11:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.337139 3169 heartbeater.cc:344] Connected to a master server at 127.2.121.189:43783
14:11:35.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.344565 3961 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "9f94afdf18ee48769d301f9bc00ff0c9" instance_seqno: 1777903873751575) as {username='kudu', principal='kudu/127.2.121.130@KRBTEST.COM'} at 127.2.121.130:50683; Asking this server to re-register.
14:11:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:35 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903894, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.189@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:35.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.350813 3169 heartbeater.cc:461] Registering TS with master...
14:11:35.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.355347 3960 ts_manager.cc:194] Registered new tserver with Master: 9f94afdf18ee48769d301f9bc00ff0c9 (127.2.121.130:34419)
14:11:35.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:35 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903894, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.189@KRBTEST.COM for kudu/127.2.121.189@KRBTEST.COM
14:11:35.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:35 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903894, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.189@KRBTEST.COM for kudu/127.2.121.188@KRBTEST.COM
14:11:35.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.544106 4015 sys_catalog.cc:455] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "a33f80b015d34ff687945c6784be125b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.545011 4015 sys_catalog.cc:458] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: This master's current role is: LEADER
14:11:35.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:35 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903886, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.131@KRBTEST.COM for kudu/127.2.121.189@KRBTEST.COM
14:11:35.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.904173 3316 heartbeater.cc:344] Connected to a master server at 127.2.121.189:43783
14:11:35.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.907441 3961 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "638f763f19db44b8b70928087de3c6c4" instance_seqno: 1777903876359054) as {username='kudu', principal='kudu/127.2.121.131@KRBTEST.COM'} at 127.2.121.131:52935; Asking this server to re-register.
14:11:35.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.909044 3316 heartbeater.cc:461] Registering TS with master...
14:11:35.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.921864 3961 ts_manager.cc:194] Registered new tserver with Master: 638f763f19db44b8b70928087de3c6c4 (127.2.121.131:42729)
14:11:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.966859 3961 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "94af139f04f64b0891ae8e49f5d705fa" instance_seqno: 1777903871343909) as {username='kudu', principal='kudu/127.2.121.129@KRBTEST.COM'} at 127.2.121.129:50581; Asking this server to re-register.
14:11:35.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.968410 3022 heartbeater.cc:461] Registering TS with master...
14:11:35.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:35.971496 3961 ts_manager.cc:194] Registered new tserver with Master: 94af139f04f64b0891ae8e49f5d705fa (127.2.121.129:38867)
14:11:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:36 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903896, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.131@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:36.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:36.053471 3198 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:36.257 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:11:36.258 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:11:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:36 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903896, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:11:36.309 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:36.309 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc
14:11:36.311 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:11:36.311 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.121.190:34499
14:11:36.312 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:11:36.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:36.313380 2534 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 3687
14:11:36.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:36.331569 4020 catalog_manager.cc:1269] Loaded cluster ID: 646f046dc1e24a52a10b732269305767
14:11:36.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:36.331861 4020 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: loading cluster ID for follower catalog manager: success
14:11:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:36.345590 4020 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: acquiring CA information for follower catalog manager: success
14:11:36.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:36.351857 4020 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:36.376 [DEBUG - kudu-nio-3] (Connection.java:239) [peer master-127.2.121.190:34499(127.2.121.190:34499)] handling channelInactive
14:11:36.376 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.2.121.190:34499(127.2.121.190:34499)] cleaning up while in state READY due to: connection closed
14:11:36.378 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:36.378 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.121.188:44851
14:11:36.378 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:11:36.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:36.379122 2534 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 3769
14:11:36.432 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:36.432 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.121.189:43783
14:11:36.432 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:11:36.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:36.433136 2534 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 3876
14:11:36.468 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:36.468 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.121.190:34499
14:11:36.469 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:11:36.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:36.471318 2534 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:36.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/logs
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data/info.pb
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.190:34499
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.190
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38001
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.190
14:11:36.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:36.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:36.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:36.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:11:36.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:36.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:36.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:36.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:36.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:36.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:36.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:36.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:36.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:11:36.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:36.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:36.867619 4029 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:11:36.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:36.868172 4029 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:36.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:36.869038 4029 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:11:36.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:36.904181 4029 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:36.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:36.904516 4029 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:11:36.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:36.904933 4029 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:11:36.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:36.905294 4029 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:36.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:36.905642 4029 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:11:36.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:36.905968 4029 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:36.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:36.906224 4029 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34683/jwks.json
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:36.952574 4029 master_runner.cc:386] Master server non-default flags:
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.190
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.190:34499
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data/info.pb
14:11:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.190
14:11:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38001
14:11:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/logs
14:11:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:11:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:11:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:11:36.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:11:36.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:36.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:36.954373 4029 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:36.955802 4029 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:36.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:36.967370 4037 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:38.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:38.411164 2378 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad880 after lost signal to thread 2253
14:11:38.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:38.412858 2378 debug-util.cc:398] Leaking SignalData structure 0x7b08000b67e0 after lost signal to thread 2381
14:11:38.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:38.370837 4036 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 4029
14:11:38.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:38.804899 4036 kernel_stack_watchdog.cc:198] Thread 4029 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
14:11:38.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:11:38.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:11:38.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:11:38.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:11:38.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:36.968735 4038 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:38.807514 4039 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1839 milliseconds
14:11:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:38.808110 4040 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:38.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:38.808765 4029 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.841s user 0.599s sys 1.057s
14:11:38.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:38.809111 4029 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.842s user 0.599s sys 1.057s
14:11:38.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:38.809387 4029 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:11:38.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:38.815898 4029 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:38.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:38.822450 4029 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:38.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:38.824285 4029 hybrid_clock.cc:648] HybridClock initialized: now 1777903898824224 us; error 70 us; skew 500 ppm
14:11:38.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:38.825377 4029 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:38.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:38 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903898, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:38.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:38.836280 4029 init.cc:377] Logged in from keytab as kudu/127.2.121.190@KRBTEST.COM (short username kudu)
14:11:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:38.841964 4029 webserver.cc:492] Webserver started at http://127.2.121.190:38001/ using document root <none> and password file <none>
14:11:38.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:38.842855 4029 fs_manager.cc:362] Metadata directory not provided
14:11:38.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:38.843118 4029 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:38.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:38.852891 4029 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.004s sys 0.000s
14:11:38.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:38.858843 4057 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:38.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:38.860188 4029 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
14:11:38.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:38.860589 4029 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:11:38.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a33f80b015d34ff687945c6784be125b"
14:11:38.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:10:59 on dist-test-slave-vb5z"
14:11:38.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:38.863071 4029 fs_report.cc:389] FS layout report
14:11:38.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:38.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:11:38.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/wal
14:11:38.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data/data
14:11:38.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:38.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:38.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:38.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:38.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:38.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:38.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:38.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:38.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:38.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:38.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:38.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:38.938026 4029 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:38.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:38.950850 4029 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:38.951539 4029 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:39.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.048579 4029 rpc_server.cc:307] RPC server started. Bound to: 127.2.121.190:34499
14:11:39.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.048732 4108 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.121.190:34499 every 8 connection(s)
14:11:39.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.051441 4029 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-0/data/info.pb
14:11:39.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.060458 2534 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 4029
14:11:39.061 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:39.061 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.121.188:44851
14:11:39.062 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.063759 2534 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/logs
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data/info.pb
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.188:44851
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.188
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33541
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.188
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:11:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:39.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.066237 4109 sys_catalog.cc:263] Verifying existing consensus state
14:11:39.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.073891 4109 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: Bootstrap starting.
14:11:39.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.116271 4109 log.cc:826] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: Log is configured to *not* fsync() on all Append() calls
14:11:39.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.136652 4109 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:11:39.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.138260 4109 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: Bootstrap complete.
14:11:39.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.163373 4109 raft_consensus.cc:359] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:39.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.165755 4109 raft_consensus.cc:740] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: a33f80b015d34ff687945c6784be125b, State: Initialized, Role: FOLLOWER
14:11:39.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.166716 4109 consensus_queue.cc:260] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:39.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.171581 4113 sys_catalog.cc:455] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:39.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.172883 4113 sys_catalog.cc:458] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: This master's current role is: FOLLOWER
14:11:39.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.173616 4109 sys_catalog.cc:565] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: configured and running, proceeding with master startup.
14:11:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.198233 4124 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:11:39.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:39 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903893, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.130@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:39 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903898, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.190@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:39.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.251192 3170 heartbeater.cc:344] Connected to a master server at 127.2.121.190:34499
14:11:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.258363 4074 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "9f94afdf18ee48769d301f9bc00ff0c9" instance_seqno: 1777903873751575) as {username='kudu', principal='kudu/127.2.121.130@KRBTEST.COM'} at 127.2.121.130:35049; Asking this server to re-register.
14:11:39.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.260039 3170 heartbeater.cc:461] Registering TS with master...
14:11:39.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.263752 4074 ts_manager.cc:194] Registered new tserver with Master: 9f94afdf18ee48769d301f9bc00ff0c9 (127.2.121.130:34419)
14:11:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.279093 4029 master_runner.cc:186] Error getting master registration for 127.2.121.189:43783: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111),
14:11:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.312485 4029 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:11:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:39.496029 4111 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:11:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:39.496716 4111 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:39.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:39.497545 4111 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:11:39.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:39.532588 4111 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:39.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:39.532909 4111 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:11:39.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:39.533294 4111 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:11:39.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:39.533528 4111 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:39.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:39.533730 4111 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:11:39.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:39.533921 4111 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:39.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:39.534070 4111 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34683/jwks.json
14:11:39.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.589511 4111 master_runner.cc:386] Master server non-default flags:
14:11:39.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:39.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:39.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:39.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:39.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.188
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.188:44851
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data/info.pb
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.188
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33541
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/logs
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.590782 4111 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.592062 4111 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:39.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:39.607918 4136 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:39.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:39 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903891, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.129@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:39.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:39 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903896, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.131@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:39.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.947623 3317 heartbeater.cc:344] Connected to a master server at 127.2.121.190:34499
14:11:39.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.950981 4074 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "638f763f19db44b8b70928087de3c6c4" instance_seqno: 1777903876359054) as {username='kudu', principal='kudu/127.2.121.131@KRBTEST.COM'} at 127.2.121.131:50987; Asking this server to re-register.
14:11:39.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.952566 3317 heartbeater.cc:461] Registering TS with master...
14:11:39.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.955911 4074 ts_manager.cc:194] Registered new tserver with Master: 638f763f19db44b8b70928087de3c6c4 (127.2.121.131:42729)
14:11:39.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.966141 3023 heartbeater.cc:344] Connected to a master server at 127.2.121.190:34499
14:11:39.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.977232 4074 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "94af139f04f64b0891ae8e49f5d705fa" instance_seqno: 1777903871343909) as {username='kudu', principal='kudu/127.2.121.129@KRBTEST.COM'} at 127.2.121.129:42957; Asking this server to re-register.
14:11:39.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.991683 3023 heartbeater.cc:461] Registering TS with master...
14:11:39.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:39.996078 4074 ts_manager.cc:194] Registered new tserver with Master: 94af139f04f64b0891ae8e49f5d705fa (127.2.121.129:38867)
14:11:40.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:40.225276 4124 catalog_manager.cc:1269] Loaded cluster ID: 646f046dc1e24a52a10b732269305767
14:11:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:40.225914 4124 catalog_manager.cc:1562] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: loading cluster ID for follower catalog manager: success
14:11:40.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:40.243000 4124 catalog_manager.cc:1584] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: acquiring CA information for follower catalog manager: success
14:11:40.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:40.264351 4124 catalog_manager.cc:1612] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:40.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:40.340859 4150 raft_consensus.cc:493] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:11:40.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:40.343283 4150 raft_consensus.cc:515] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:40.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:40.352566 4061 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:40.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:40.369814 4150 leader_election.cc:290] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783), 3e861f07d8954c4fa14d0eb4457d7f28 (127.2.121.188:44851)
14:11:40.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:40.376252 4060 leader_election.cc:336] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 3e861f07d8954c4fa14d0eb4457d7f28 (127.2.121.188:44851): Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:40.384233 4061 leader_election.cc:336] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783): Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:40.385075 4061 leader_election.cc:304] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a33f80b015d34ff687945c6784be125b; no voters: 3e861f07d8954c4fa14d0eb4457d7f28, 5cbbf08fc96d4a45af7a03eec236c560
14:11:40.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:40.386365 4150 raft_consensus.cc:2749] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
14:11:40.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:40.902946 2518 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9c20 after lost signal to thread 2393
14:11:40.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:40.904487 2518 debug-util.cc:398] Leaking SignalData structure 0x7b08000baae0 after lost signal to thread 2521
14:11:41.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:41 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903901, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.129@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:41.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:41.051856 2904 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:42.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.153443 4161 raft_consensus.cc:493] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:11:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.154016 4161 raft_consensus.cc:515] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:42.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.177870 4161 leader_election.cc:290] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783), 3e861f07d8954c4fa14d0eb4457d7f28 (127.2.121.188:44851)
14:11:42.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:42.208736 4061 leader_election.cc:336] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783): Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:42.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:42.212447 4060 leader_election.cc:336] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 3e861f07d8954c4fa14d0eb4457d7f28 (127.2.121.188:44851): Network error: Client connection negotiation failed: client connection to 127.2.121.188:44851: connect: Connection refused (error 111)
14:11:42.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.213027 4060 leader_election.cc:304] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a33f80b015d34ff687945c6784be125b; no voters: 3e861f07d8954c4fa14d0eb4457d7f28, 5cbbf08fc96d4a45af7a03eec236c560
14:11:42.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.216362 4161 raft_consensus.cc:2749] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
14:11:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:41.006698 4134 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 4111
14:11:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:42.586994 4134 kernel_stack_watchdog.cc:198] Thread 4111 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
14:11:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:11:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:11:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:11:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:11:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:39.609519 4135 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:42.589962 4111 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.983s user 0.500s sys 0.764s
14:11:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:42.590301 4138 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:42.590692 4111 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.984s user 0.501s sys 0.765s
14:11:42.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:42.594511 4137 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2987 milliseconds
14:11:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.594558 4111 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:11:42.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.596577 4111 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.605535 4111 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:42.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.607120 4111 hybrid_clock.cc:648] HybridClock initialized: now 1777903902607048 us; error 86 us; skew 500 ppm
14:11:42.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.611446 4111 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:42.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:42 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903902, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:42.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.621665 4111 init.cc:377] Logged in from keytab as kudu/127.2.121.188@KRBTEST.COM (short username kudu)
14:11:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.645869 4111 webserver.cc:492] Webserver started at http://127.2.121.188:33541/ using document root <none> and password file <none>
14:11:42.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.647723 4111 fs_manager.cc:362] Metadata directory not provided
14:11:42.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.648051 4111 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:42.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.661281 4111 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.005s sys 0.001s
14:11:42.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.692392 4173 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:42.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.694214 4111 fs_manager.cc:730] Time spent opening block manager: real 0.030s user 0.015s sys 0.009s
14:11:42.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.694693 4111 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:42.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3e861f07d8954c4fa14d0eb4457d7f28"
14:11:42.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:04 on dist-test-slave-vb5z"
14:11:42.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.697866 4111 fs_report.cc:389] FS layout report
14:11:42.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:42.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:42.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/wal
14:11:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data/data
14:11:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:42.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:42.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:42.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:42.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:42.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:42.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.773327 4111 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:42.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.782220 4111 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:42.782892 4111 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:43.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.006305 4111 rpc_server.cc:307] RPC server started. Bound to: 127.2.121.188:44851
14:11:43.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.006376 4226 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.121.188:44851 every 8 connection(s)
14:11:43.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.013118 4111 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-2/data/info.pb
14:11:43.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.013824 2534 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 4111
14:11:43.014 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:43.014 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.121.189:43783
14:11:43.016 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:11:43.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.018249 2534 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:43.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:11:43.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:43.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data
14:11:43.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:43.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:43.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:43.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:43.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:43.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/logs
14:11:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data/info.pb
14:11:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.189:43783
14:11:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.189
14:11:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36363
14:11:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.189
14:11:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:11:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:11:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.074154 4227 sys_catalog.cc:263] Verifying existing consensus state
14:11:43.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.092257 4227 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: Bootstrap starting.
14:11:43.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.182152 4227 log.cc:826] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: Log is configured to *not* fsync() on all Append() calls
14:11:43.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.213712 4227 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:11:43.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.215034 4227 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: Bootstrap complete.
14:11:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.254320 4227 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:43.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.258239 4227 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3e861f07d8954c4fa14d0eb4457d7f28, State: Initialized, Role: FOLLOWER
14:11:43.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.260605 4227 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:43.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.274660 4227 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: configured and running, proceeding with master startup.
14:11:43.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.280085 4232 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:43.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.281332 4232 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: This master's current role is: FOLLOWER
14:11:43.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:43.349984 4239 negotiation.cc:336] Failed RPC negotiation. Trace:
14:11:43.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:43.320723 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.2.121.130:58107 (local address 127.2.121.188:44851)
14:11:43.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:43.339895 (+ 19172us) server_negotiation.cc:207] Beginning negotiation
14:11:43.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:43.339964 (+ 69us) server_negotiation.cc:400] Waiting for connection header
14:11:43.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:43.340065 (+ 101us) server_negotiation.cc:408] Connection header received
14:11:43.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:43.343325 (+ 3260us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:11:43.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:43.343383 (+ 58us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:11:43.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:43.344065 (+ 682us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:11:43.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:43.344581 (+ 516us) server_negotiation.cc:227] Negotiated authn=SASL
14:11:43.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:43.347813 (+ 3232us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.2.121.130:58107: BlockingRecv error: recv got EOF from 127.2.121.130:58107 (error 108)
14:11:43.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":17894,"thread_start_us":17336,"threads_started":1}
14:11:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.452711 4245 catalog_manager.cc:1269] Loaded cluster ID: 646f046dc1e24a52a10b732269305767
14:11:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.452997 4245 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: loading cluster ID for follower catalog manager: success
14:11:43.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.461712 4245 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: acquiring CA information for follower catalog manager: success
14:11:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.468115 4245 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:43.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:43 dist-test-slave-vb5z krb5kdc[2546](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903902, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.188@KRBTEST.COM for kudu/127.2.121.190@KRBTEST.COM
14:11:43.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:43 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903903, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.130@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:43.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.496631 3051 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:43.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.511041 4111 master_runner.cc:186] Error getting master registration for 127.2.121.189:43783: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111),
14:11:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.547308 4111 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:11:43.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:43.655035 4229 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:11:43.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:43.656533 4229 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:43.658125 4229 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:11:43.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:43.731010 4229 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:43.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:43.731851 4229 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:11:43.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:43.732617 4229 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:11:43.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:43.733268 4229 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:43.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:43.733794 4229 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:11:43.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:43.734299 4229 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:43.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:43.734804 4229 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34683/jwks.json
14:11:43.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.820796 4229 master_runner.cc:386] Master server non-default flags:
14:11:43.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:43.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.121.148:45309
14:11:43.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:43.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data
14:11:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.121.190:34499,127.2.121.189:43783,127.2.121.188:44851
14:11:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/krb5kdc/kudu.keytab
14:11:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.121.189
14:11:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.121.189:43783
14:11:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34683/jwks.json
14:11:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data/info.pb
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.121.189
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36363
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/logs
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:11:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:43.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.825891 4229 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:43.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.827775 4229 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:43.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:43.856664 4254 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:43.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.964040 3315 heartbeater.cc:344] Connected to a master server at 127.2.121.188:44851
14:11:43.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.965377 3021 heartbeater.cc:344] Connected to a master server at 127.2.121.188:44851
14:11:43.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.970814 4190 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "638f763f19db44b8b70928087de3c6c4" instance_seqno: 1777903876359054) as {username='kudu', principal='kudu/127.2.121.131@KRBTEST.COM'} at 127.2.121.131:57419; Asking this server to re-register.
14:11:43.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.973130 3315 heartbeater.cc:461] Registering TS with master...
14:11:43.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.973294 4189 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "94af139f04f64b0891ae8e49f5d705fa" instance_seqno: 1777903871343909) as {username='kudu', principal='kudu/127.2.121.129@KRBTEST.COM'} at 127.2.121.129:47583; Asking this server to re-register.
14:11:43.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.976917 3021 heartbeater.cc:461] Registering TS with master...
14:11:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.981840 4189 ts_manager.cc:194] Registered new tserver with Master: 638f763f19db44b8b70928087de3c6c4 (127.2.121.131:42729)
14:11:43.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:43.984735 4189 ts_manager.cc:194] Registered new tserver with Master: 94af139f04f64b0891ae8e49f5d705fa (127.2.121.129:38867)
14:11:44.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.327288 4029 master_runner.cc:186] Error getting master registration for 127.2.121.189:43783: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111),
14:11:44.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.328130 4029 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:11:44.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.370071 3168 heartbeater.cc:344] Connected to a master server at 127.2.121.188:44851
14:11:44.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.373687 4189 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "9f94afdf18ee48769d301f9bc00ff0c9" instance_seqno: 1777903873751575) as {username='kudu', principal='kudu/127.2.121.130@KRBTEST.COM'} at 127.2.121.130:32903; Asking this server to re-register.
14:11:44.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.376116 3168 heartbeater.cc:461] Registering TS with master...
14:11:44.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.379628 4189 ts_manager.cc:194] Registered new tserver with Master: 9f94afdf18ee48769d301f9bc00ff0c9 (127.2.121.130:34419)
14:11:44.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.547960 4267 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:11:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.549433 4267 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:44.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.566417 4084 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "a33f80b015d34ff687945c6784be125b" is_pre_election: true
14:11:44.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.567296 4084 raft_consensus.cc:2468] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3e861f07d8954c4fa14d0eb4457d7f28 in term 3.
14:11:44.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.568845 4177 leader_election.cc:304] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3e861f07d8954c4fa14d0eb4457d7f28, a33f80b015d34ff687945c6784be125b; no voters:
14:11:44.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.572932 4267 leader_election.cc:290] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers a33f80b015d34ff687945c6784be125b (127.2.121.190:34499), 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783)
14:11:44.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.573473 4267 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 3 FOLLOWER]: Leader pre-election won for term 4
14:11:44.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.573814 4267 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:11:44.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.574150 4267 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 3 FOLLOWER]: Advancing to term 4
14:11:44.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:44.590016 4177 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:44.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:44.603749 4177 leader_election.cc:336] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783): Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:44.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.594811 4267 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:44.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.606776 4084 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "a33f80b015d34ff687945c6784be125b"
14:11:44.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.607369 4084 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 3 FOLLOWER]: Advancing to term 4
14:11:44.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.616652 4084 raft_consensus.cc:2468] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3e861f07d8954c4fa14d0eb4457d7f28 in term 4.
14:11:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.617955 4177 leader_election.cc:304] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3e861f07d8954c4fa14d0eb4457d7f28, a33f80b015d34ff687945c6784be125b; no voters:
14:11:44.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.618865 4268 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 4 FOLLOWER]: Leader election won for term 4
14:11:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:44.634960 4177 leader_election.cc:336] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783): Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111)
14:11:44.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.636044 4267 leader_election.cc:290] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [CANDIDATE]: Term 4 election: Requested vote from peers a33f80b015d34ff687945c6784be125b (127.2.121.190:34499), 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783)
14:11:44.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.640890 4268 raft_consensus.cc:697] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [term 4 LEADER]: Becoming Leader. State: Replica: 3e861f07d8954c4fa14d0eb4457d7f28, State: Running, Role: LEADER
14:11:44.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.641919 4268 consensus_queue.cc:237] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:44.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.662683 4267 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3e861f07d8954c4fa14d0eb4457d7f28. Latest consensus state: current_term: 4 leader_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:44.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.663453 4267 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: This master's current role is: LEADER
14:11:44.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.671833 4273 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
14:11:44.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.699709 4273 catalog_manager.cc:1494] Initializing Kudu cluster ID...
14:11:44.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.709996 4273 catalog_manager.cc:1269] Loaded cluster ID: 646f046dc1e24a52a10b732269305767
14:11:44.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.719655 4273 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
14:11:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.726548 4273 catalog_manager.cc:1514] Loading token signing keys...
14:11:44.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.737594 4273 catalog_manager.cc:6055] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28: Loaded TSK: 0
14:11:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:44.747910 4273 catalog_manager.cc:1524] Initializing in-progress tserver states...
14:11:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:45.083336 4177 consensus_peers.cc:597] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 -> Peer 5cbbf08fc96d4a45af7a03eec236c560 (127.2.121.189:43783): Couldn't send request to peer 5cbbf08fc96d4a45af7a03eec236c560. Status: Network error: Client connection negotiation failed: client connection to 127.2.121.189:43783: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:11:45.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.150638 4084 raft_consensus.cc:1275] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [term 4 FOLLOWER]: Refusing update from remote peer 3e861f07d8954c4fa14d0eb4457d7f28: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
14:11:45.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.152474 4267 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
14:11:45.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.185600 4276 sys_catalog.cc:455] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3e861f07d8954c4fa14d0eb4457d7f28. Latest consensus state: current_term: 4 leader_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:45.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.187258 4276 sys_catalog.cc:458] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: This master's current role is: FOLLOWER
14:11:45.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.198491 4276 sys_catalog.cc:455] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:45.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.200100 4276 sys_catalog.cc:458] T 00000000000000000000000000000000 P a33f80b015d34ff687945c6784be125b [sys.catalog]: This master's current role is: FOLLOWER
14:11:45.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.201669 4267 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:45.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.202360 4267 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3e861f07d8954c4fa14d0eb4457d7f28 [sys.catalog]: This master's current role is: LEADER
14:11:45.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:45.255329 4253 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 4229
14:11:45.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:43.862358 4255 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:45.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:45.864763 4229 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.004s user 0.510s sys 0.784s
14:11:45.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:45.865366 4229 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.005s user 0.510s sys 0.784s
14:11:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:45.864306 4253 kernel_stack_watchdog.cc:198] Thread 4229 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 393ms:
14:11:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:11:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:11:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:11:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:11:45.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.895983 4229 server_base.cc:1061] running on GCE node
14:11:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:45.897953 4284 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:45.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.899669 4229 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.902850 4229 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.904309 4229 hybrid_clock.cc:648] HybridClock initialized: now 1777903905904210 us; error 104 us; skew 500 ppm
14:11:45.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.905359 4229 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:45.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:45 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903905, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:45.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.913975 4229 init.cc:377] Logged in from keytab as kudu/127.2.121.189@KRBTEST.COM (short username kudu)
14:11:45.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.928829 4229 webserver.cc:492] Webserver started at http://127.2.121.189:36363/ using document root <none> and password file <none>
14:11:45.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.930245 4229 fs_manager.cc:362] Metadata directory not provided
14:11:45.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.930578 4229 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.942960 4229 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.005s sys 0.001s
14:11:45.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.949342 4290 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:45.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.950857 4229 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
14:11:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.951300 4229 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5cbbf08fc96d4a45af7a03eec236c560"
14:11:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:11:01 on dist-test-slave-vb5z"
14:11:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.954399 4229 fs_report.cc:389] FS layout report
14:11:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/wal
14:11:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data/data
14:11:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:45.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.990813 3315 heartbeater.cc:499] Master 127.2.121.188:44851 was elected leader, sending a full tablet report...
14:11:45.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:45.993233 3021 heartbeater.cc:499] Master 127.2.121.188:44851 was elected leader, sending a full tablet report...
14:11:46.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.046720 4229 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:46.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:11:46 dist-test-slave-vb5z krb5kdc[2546](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777903906, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.121.131@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.057587 3198 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.058578 4229 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:46.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.059128 4229 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:46.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.160676 4229 rpc_server.cc:307] RPC server started. Bound to: 127.2.121.189:43783
14:11:46.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.160761 4344 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.121.189:43783 every 8 connection(s)
14:11:46.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.163843 4229 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster1883777674088405996/master-1/data/info.pb
14:11:46.172 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:46.173 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.121.190:34499(127.2.121.190:34499)] connecting to peer
14:11:46.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.171289 2534 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 4229
14:11:46.183 [DEBUG - kudu-nio-4] (Connection.java:757) [peer master-127.2.121.190:34499(127.2.121.190:34499)] Successfully connected to peer
14:11:46.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.186105 4345 sys_catalog.cc:263] Verifying existing consensus state
14:11:46.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.195256 4345 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: Bootstrap starting.
14:11:46.197 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.2.121.190:34499(127.2.121.190:34499)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:11:46.200 [DEBUG - kudu-nio-4] (Connection.java:239) [peer master-127.2.121.190:34499(127.2.121.190:34499)] handling channelInactive
14:11:46.201 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.121.190:34499(127.2.121.190:34499)] disconnecting while in state TERMINATED
14:11:46.202 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.121.189:43783(127.2.121.189:43783)] disconnecting while in state TERMINATED
14:11:46.202 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.121.188:44851(127.2.121.188:44851)] disconnecting while in state TERMINATED
14:11:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:11:46.202551 4348 negotiation.cc:336] Failed RPC negotiation. Trace:
14:11:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:46.179278 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.0.0.1:49236 (local address 127.2.121.190:34499)
14:11:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:46.195069 (+ 15791us) server_negotiation.cc:207] Beginning negotiation
14:11:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:46.195097 (+ 28us) server_negotiation.cc:400] Waiting for connection header
14:11:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:46.195155 (+ 58us) server_negotiation.cc:408] Connection header received
14:11:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:46.195313 (+ 158us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:11:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:46.195341 (+ 28us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:11:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:46.195625 (+ 284us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:11:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:46.195841 (+ 216us) server_negotiation.cc:227] Negotiated authn=SASL
14:11:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0504 14:11:46.201048 (+ 5207us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:49236: BlockingRecv error: recv got EOF from 127.0.0.1:49236 (error 108)
14:11:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":15150,"thread_start_us":14244,"threads_started":1}
14:11:46.204 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:11:46.211 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:11:46.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.214423 2534 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 2885
14:11:46.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.256525 2534 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 3035
14:11:46.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.283368 4345 log.cc:826] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: Log is configured to *not* fsync() on all Append() calls
14:11:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.304219 2534 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 3182
14:11:46.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.307730 4345 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:11:46.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.308965 4345 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560: Bootstrap complete.
14:11:46.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.345198 2534 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 4029
14:11:46.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.349872 4345 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:46.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.353536 4345 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5cbbf08fc96d4a45af7a03eec236c560, State: Initialized, Role: FOLLOWER
14:11:46.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.354593 4345 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } }
14:11:46.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.358013 4356 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a33f80b015d34ff687945c6784be125b" member_type: VOTER last_known_addr { host: "127.2.121.190" port: 34499 } } peers { permanent_uuid: "5cbbf08fc96d4a45af7a03eec236c560" member_type: VOTER last_known_addr { host: "127.2.121.189" port: 43783 } } peers { permanent_uuid: "3e861f07d8954c4fa14d0eb4457d7f28" member_type: VOTER last_known_addr { host: "127.2.121.188" port: 44851 } } }
14:11:46.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.359990 4356 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: This master's current role is: FOLLOWER
14:11:46.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.360241 4345 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5cbbf08fc96d4a45af7a03eec236c560 [sys.catalog]: configured and running, proceeding with master startup.
14:11:46.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.401768 2534 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 4229
14:11:46.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:46.425752 2534 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 4111
14:11:47.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-04T14:11:47Z chronyd exiting
14:11:48.709 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:11:48.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:48.712321 1808 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 2089
14:11:48.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:48.741526 1808 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 2251
14:11:48.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:48.770795 1808 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 2392
14:11:48.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:48.802043 1808 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 1820
14:11:48.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:48.835358 1808 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 1890
14:11:48.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:11:48.872046 1808 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 1965
14:11:48.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-04T14:11:48Z chronyd exiting
14:11:50.010 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more