13:59:55.979 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
13:59:55.992 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskTlaXut/test-tmp
13:59:56.000 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
13:59:56.004 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskTlaXut/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
13:59:56.238 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
13:59:56.451 [DEBUG - main] (MiniKuduCluster.java:195) Response:
13:59:56.452 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
13:59:56.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:56.458705 28175 mini_oidc.cc:150] Starting JWKS server
13:59:56.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:56.462882 28175 webserver.cc:533] Webserver started at http://127.0.0.1:35257/ using document root <none> and password file <none>
13:59:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:56.463702 28175 mini_oidc.cc:186] Starting OIDC Discovery server
13:59:56.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:56.465842 28175 webserver.cc:533] Webserver started at http://127.0.0.1:40865/ using document root <none> and password file <none>
13:59:56.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T13:59:56Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
13:59:56.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T13:59:56Z Disabled control of system clock
13:59:56.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:56.528838 28175 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
13:59:56.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
13:59:56.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-0/wal
13:59:56.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-0/data
13:59:56.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:59:56.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:59:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:59:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-0/logs
13:59:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-0/data/info.pb
13:59:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:59:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:59:56.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:59:56.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:59:56.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:56.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:56.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:56.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:59:56.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:59:56.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:56.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:56.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.131.254:44375
13:59:56.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.131.254
13:59:56.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:56.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.131.212:43191
13:59:56.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:56.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:56.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:56.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:56.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.131.254:44375,127.27.131.253:45199,127.27.131.252:44291
13:59:56.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:56.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35257/jwks.json with env {}
13:59:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:56.839632 28196 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
13:59:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:56.840214 28196 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:59:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:56.840757 28196 flags.cc:432] Enabled unsafe flag: --never_fsync=true
13:59:56.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:56.874579 28196 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
13:59:56.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:56.874886 28196 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
13:59:56.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:56.875098 28196 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
13:59:56.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:56.875296 28196 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
13:59:56.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:56.875553 28196 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
13:59:56.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:56.875712 28196 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35257/jwks.json
13:59:56.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:56.914947 28196 master_runner.cc:387] Master server non-default flags:
13:59:56.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-0/logs
13:59:56.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:56.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:59:56.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:56.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.131.212:43191
13:59:56.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:56.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:56.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-0/data
13:59:56.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-0/wal
13:59:56.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:56.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.131.254:44375,127.27.131.253:45199,127.27.131.252:44291
13:59:56.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:56.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:56.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:56.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.131.254:44375
13:59:56.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:56.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:59:56.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:56.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35257/jwks.json
13:59:56.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:56.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:56.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-0/data/info.pb
13:59:56.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.131.254
13:59:56.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:56.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:59:56.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:56.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:59:56.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:59:56.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:56.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:59:56.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:59:56.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
13:59:56.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 70c51312f1186a8ae9f46647160b7730123f7601
13:59:56.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:59:56.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 13:43:22 UTC on e7f111948823
13:59:56.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13307
13:59:56.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:59:56.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:56.916702 28196 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:59:56.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:56.918577 28196 file_cache.cc:504] Constructed file cache file cache with capacity 419430
13:59:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:56.933583 28202 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:58.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:56.936792 28203 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:58.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.048213 28204 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1109 milliseconds
13:59:58.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.049129 28205 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:58.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.052006 28196 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.115s user 0.377s sys 0.734s
13:59:58.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.052286 28196 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.116s user 0.377s sys 0.734s
13:59:58.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.052552 28196 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:59:58.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.054021 28196 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:59:58.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.056818 28196 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:59:58.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.058543 28196 hybrid_clock.cc:648] HybridClock initialized: now 1786543198058507 us; error 198 us; skew 500 ppm
13:59:58.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.059693 28196 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:59:58.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.068197 28196 webserver.cc:533] Webserver started at http://127.27.131.254:33303/ using document root <none> and password file <none>
13:59:58.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.069976 28196 fs_manager.cc:362] Metadata directory not provided
13:59:58.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.070235 28196 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:59:58.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.070848 28196 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
13:59:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.075522 28196 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-0/data/instance:
13:59:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dbbb07ceed49450188856ccbeb983ad0"
13:59:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 13:59:58 on dist-test-slave-vpvm"
13:59:58.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.076848 28196 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-0/wal/instance:
13:59:58.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dbbb07ceed49450188856ccbeb983ad0"
13:59:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 13:59:58 on dist-test-slave-vpvm"
13:59:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.085454 28196 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.010s sys 0.000s
13:59:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.091869 28221 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:59:58.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.093546 28196 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.001s
13:59:58.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.093941 28196 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-0/data,/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-0/wal
13:59:58.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dbbb07ceed49450188856ccbeb983ad0"
13:59:58.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 13:59:58 on dist-test-slave-vpvm"
13:59:58.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.094460 28196 fs_report.cc:389] FS layout report
13:59:58.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:59:58.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-0/wal
13:59:58.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-0/wal
13:59:58.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-0/data/data
13:59:58.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:59:58.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:59:58.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:59:58.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:59:58.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:59:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:59:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:59:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:59:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:59:58.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:59:58.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:59:58.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.178222 28196 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:59:58.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.180011 28196 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:59:58.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.180552 28196 kserver.cc:163] Server-wide thread pool size limit: 3276
13:59:58.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.252445 28196 rpc_server.cc:307] RPC server started. Bound to: 127.27.131.254:44375
13:59:58.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.252554 28291 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.131.254:44375 every 8 connection(s)
13:59:58.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.255698 28196 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-0/data/info.pb
13:59:58.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.261788 28292 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:59:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.265784 28175 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu as pid 28196
13:59:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.266198 28175 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-0/wal/instance
13:59:58.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.269591 28175 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
13:59:58.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
13:59:58.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-1/wal
13:59:58.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-1/data
13:59:58.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:59:58.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:59:58.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:59:58.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:58.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:58.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-1/logs
13:59:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-1/data/info.pb
13:59:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:59:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:59:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:59:58.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:59:58.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:58.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:58.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:58.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:59:58.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:59:58.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:58.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:58.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.131.253:45199
13:59:58.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.131.253
13:59:58.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:58.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.131.212:43191
13:59:58.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:58.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:58.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:58.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:58.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.131.254:44375,127.27.131.253:45199,127.27.131.252:44291
13:59:58.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:58.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35257/jwks.json with env {}
13:59:58.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.274353 28292 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } has no permanent_uuid. Determining permanent_uuid...
13:59:58.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.301822 28292 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } has no permanent_uuid. Determining permanent_uuid...
13:59:58.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.304414 28223 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.131.253:45199: connect: Connection refused (error 111)
13:59:58.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.308681 28292 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.253:45199: Network error: Client connection negotiation failed: client connection to 127.27.131.253:45199: connect: Connection refused (error 111)
13:59:58.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.358208 28292 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } attempt: 1
13:59:58.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.363126 28292 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.253:45199: Network error: Client connection negotiation failed: client connection to 127.27.131.253:45199: connect: Connection refused (error 111)
13:59:58.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.431563 28292 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } attempt: 2
13:59:58.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.436184 28292 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.253:45199: Network error: Client connection negotiation failed: client connection to 127.27.131.253:45199: connect: Connection refused (error 111)
13:59:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.527580 28292 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } attempt: 3
13:59:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.531951 28292 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.253:45199: Network error: Client connection negotiation failed: client connection to 127.27.131.253:45199: connect: Connection refused (error 111)
13:59:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.590231 28294 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
13:59:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.590914 28294 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:59:58.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.591691 28294 flags.cc:432] Enabled unsafe flag: --never_fsync=true
13:59:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.625787 28294 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
13:59:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.626171 28294 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
13:59:58.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.626509 28294 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
13:59:58.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.626849 28294 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
13:59:58.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.627207 28294 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
13:59:58.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.627496 28294 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35257/jwks.json
13:59:58.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.665158 28294 master_runner.cc:387] Master server non-default flags:
13:59:58.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-1/logs
13:59:58.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:58.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:59:58.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:58.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.131.212:43191
13:59:58.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:58.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:58.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-1/data
13:59:58.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-1/wal
13:59:58.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:58.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.131.254:44375,127.27.131.253:45199,127.27.131.252:44291
13:59:58.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:58.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:58.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:58.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.131.253:45199
13:59:58.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:58.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:59:58.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:58.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35257/jwks.json
13:59:58.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:58.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:58.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-1/data/info.pb
13:59:58.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.131.253
13:59:58.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:58.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:59:58.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:58.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:59:58.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:59:58.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:58.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:59:58.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:59:58.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
13:59:58.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 70c51312f1186a8ae9f46647160b7730123f7601
13:59:58.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:59:58.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 13:43:22 UTC on e7f111948823
13:59:58.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13307
13:59:58.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:59:58.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.666987 28294 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:59:58.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.669486 28294 file_cache.cc:504] Constructed file cache file cache with capacity 419430
13:59:58.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.675323 28292 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } attempt: 4
13:59:58.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.681074 28292 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.253:45199: Network error: Client connection negotiation failed: client connection to 127.27.131.253:45199: connect: Connection refused (error 111)
13:59:58.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.684352 28309 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:58.980567 28292 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } attempt: 5
13:59:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.987047 28292 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.253:45199: Network error: Client connection negotiation failed: client connection to 127.27.131.253:45199: connect: Connection refused (error 111)
13:59:59.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.534596 28292 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } attempt: 6
13:59:59.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:59.544505 28292 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.253:45199: Network error: Client connection negotiation failed: client connection to 127.27.131.253:45199: connect: Connection refused (error 111)
13:59:59.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.687078 28310 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:59.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:58.689884 28312 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:59.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 13:59:59.833387 28311 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1144 milliseconds
13:59:59.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.833518 28294 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:59:59.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.835209 28294 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:59:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.838512 28294 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:59:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.839993 28294 hybrid_clock.cc:648] HybridClock initialized: now 1786543199839946 us; error 63 us; skew 500 ppm
13:59:59.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.840843 28294 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:59:59.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.847602 28294 webserver.cc:533] Webserver started at http://127.27.131.253:34475/ using document root <none> and password file <none>
13:59:59.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.848572 28294 fs_manager.cc:362] Metadata directory not provided
13:59:59.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.848768 28294 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:59:59.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.849247 28294 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
13:59:59.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.854136 28294 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-1/data/instance:
13:59:59.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "572befe5b131437d99e612431649b95b"
13:59:59.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 13:59:59 on dist-test-slave-vpvm"
13:59:59.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.855440 28294 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-1/wal/instance:
13:59:59.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "572befe5b131437d99e612431649b95b"
13:59:59.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 13:59:59 on dist-test-slave-vpvm"
13:59:59.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.862939 28294 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.003s sys 0.005s
13:59:59.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.869019 28327 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:59:59.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.870440 28294 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
13:59:59.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.870757 28294 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-1/data,/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-1/wal
13:59:59.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "572befe5b131437d99e612431649b95b"
13:59:59.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 13:59:59 on dist-test-slave-vpvm"
13:59:59.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.871093 28294 fs_report.cc:389] FS layout report
13:59:59.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:59:59.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-1/wal
13:59:59.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-1/wal
13:59:59.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-1/data/data
13:59:59.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:59:59.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:59:59.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:59:59.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:59:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:59:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:59:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:59:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:59:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:59:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:59:59.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:59:59.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.942742 28294 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:59:59.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.944645 28294 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:59:59.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 13:59:59.945254 28294 kserver.cc:163] Server-wide thread pool size limit: 3276
14:00:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.038499 28294 rpc_server.cc:307] RPC server started. Bound to: 127.27.131.253:45199
14:00:00.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.038573 28401 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.131.253:45199 every 8 connection(s)
14:00:00.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.042235 28294 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-1/data/info.pb
14:00:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.048429 28403 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:00:00.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.052448 28175 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu as pid 28294
14:00:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.052933 28175 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-1/wal/instance
14:00:00.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.055948 28175 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:00.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:00.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-2/wal
14:00:00.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-2/data
14:00:00.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:00:00.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:00:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:00:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-2/logs
14:00:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-2/data/info.pb
14:00:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:00:00.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:00:00.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:00:00.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:00:00.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:00.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:00.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:00.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:00:00.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:00:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:00:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:00:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.131.252:44291
14:00:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.131.252
14:00:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.131.212:43191
14:00:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:00:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.131.254:44375,127.27.131.253:45199,127.27.131.252:44291
14:00:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:00:00.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35257/jwks.json with env {}
14:00:00.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.063275 28403 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } has no permanent_uuid. Determining permanent_uuid...
14:00:00.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.086104 28403 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } has no permanent_uuid. Determining permanent_uuid...
14:00:00.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.103045 28403 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } has no permanent_uuid. Determining permanent_uuid...
14:00:00.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.105611 28332 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.131.252:44291: connect: Connection refused (error 111)
14:00:00.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.110445 28403 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.252:44291: Network error: Client connection negotiation failed: client connection to 127.27.131.252:44291: connect: Connection refused (error 111)
14:00:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.160064 28403 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } attempt: 1
14:00:00.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.165028 28403 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.252:44291: Network error: Client connection negotiation failed: client connection to 127.27.131.252:44291: connect: Connection refused (error 111)
14:00:00.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.233559 28403 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } attempt: 2
14:00:00.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.238587 28403 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.252:44291: Network error: Client connection negotiation failed: client connection to 127.27.131.252:44291: connect: Connection refused (error 111)
14:00:00.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.330087 28403 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } attempt: 3
14:00:00.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.334648 28403 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.252:44291: Network error: Client connection negotiation failed: client connection to 127.27.131.252:44291: connect: Connection refused (error 111)
14:00:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.426357 28405 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:00:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.426962 28405 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:00:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.427428 28405 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:00:00.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.461114 28405 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:00:00.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.461434 28405 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:00:00.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.461635 28405 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:00:00.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.461829 28405 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:00:00.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.462080 28405 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:00:00.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.462332 28405 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35257/jwks.json
14:00:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.478221 28403 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } attempt: 4
14:00:00.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.482604 28403 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.252:44291: Network error: Client connection negotiation failed: client connection to 127.27.131.252:44291: connect: Connection refused (error 111)
14:00:00.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.501184 28405 master_runner.cc:387] Master server non-default flags:
14:00:00.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-2/logs
14:00:00.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:00.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:00:00.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:00.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.131.212:43191
14:00:00.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:00.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:00.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-2/data
14:00:00.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-2/wal
14:00:00.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:00:00.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.131.254:44375,127.27.131.253:45199,127.27.131.252:44291
14:00:00.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:00.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:00.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:00:00.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.131.252:44291
14:00:00.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:00:00.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:00:00.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:00:00.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35257/jwks.json
14:00:00.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:00.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:00.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-2/data/info.pb
14:00:00.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.131.252
14:00:00.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:00.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:00:00.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:00.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:00:00.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:00:00.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:00.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:00:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:00:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:00:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 70c51312f1186a8ae9f46647160b7730123f7601
14:00:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:00:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 13:43:22 UTC on e7f111948823
14:00:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13307
14:00:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:00:00.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.503019 28405 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:00:00.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.505612 28405 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:00:00.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.519294 28422 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:00.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.605072 28292 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } attempt: 7
14:00:00.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.637624 28292 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } has no permanent_uuid. Determining permanent_uuid...
14:00:00.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.654852 28292 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.252:44291: Network error: Client connection negotiation failed: client connection to 127.27.131.252:44291: connect: Connection refused (error 111)
14:00:00.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.713413 28292 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } attempt: 1
14:00:00.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.720861 28292 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.252:44291: Network error: Client connection negotiation failed: client connection to 127.27.131.252:44291: connect: Connection refused (error 111)
14:00:00.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.782058 28403 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } attempt: 5
14:00:00.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.789788 28403 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.252:44291: Network error: Client connection negotiation failed: client connection to 127.27.131.252:44291: connect: Connection refused (error 111)
14:00:00.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.802366 28292 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } attempt: 2
14:00:00.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.810664 28292 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.252:44291: Network error: Client connection negotiation failed: client connection to 127.27.131.252:44291: connect: Connection refused (error 111)
14:00:00.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:00.896169 28292 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } attempt: 3
14:00:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.903910 28292 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.252:44291: Network error: Client connection negotiation failed: client connection to 127.27.131.252:44291: connect: Connection refused (error 111)
14:00:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.044409 28292 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } attempt: 4
14:00:01.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:01.064951 28292 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.252:44291: Network error: Client connection negotiation failed: client connection to 127.27.131.252:44291: connect: Connection refused (error 111)
14:00:01.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.337332 28403 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } attempt: 6
14:00:01.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.348469 28292 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } attempt: 5
14:00:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:01.355995 28292 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.252:44291: Network error: Client connection negotiation failed: client connection to 127.27.131.252:44291: connect: Connection refused (error 111)
14:00:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:01.364418 28403 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.252:44291: Network error: Client connection negotiation failed: client connection to 127.27.131.252:44291: connect: Connection refused (error 111)
14:00:01.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:00.520597 28423 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:01.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:01.804311 28427 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1282 milliseconds
14:00:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:01.805212 28429 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:01.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:01.807791 28405 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.288s user 0.552s sys 0.729s
14:00:01.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:01.808152 28405 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.288s user 0.552s sys 0.729s
14:00:01.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.808423 28405 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:00:01.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.810861 28405 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:00:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.814101 28405 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:00:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.815531 28405 hybrid_clock.cc:648] HybridClock initialized: now 1786543201815445 us; error 83 us; skew 500 ppm
14:00:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.816653 28405 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:00:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.825699 28405 webserver.cc:533] Webserver started at http://127.27.131.252:35473/ using document root <none> and password file <none>
14:00:01.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.827062 28405 fs_manager.cc:362] Metadata directory not provided
14:00:01.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.827358 28405 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:00:01.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.827946 28405 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:00:01.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.834537 28405 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-2/data/instance:
14:00:01.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9e7e867b3f34e81b9c9d465e50ab395"
14:00:01.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:01 on dist-test-slave-vpvm"
14:00:01.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.836077 28405 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-2/wal/instance:
14:00:01.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9e7e867b3f34e81b9c9d465e50ab395"
14:00:01.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:01 on dist-test-slave-vpvm"
14:00:01.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.846302 28405 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.006s sys 0.005s
14:00:01.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.854185 28446 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:00:01.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.855634 28405 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
14:00:01.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.856035 28405 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-2/data,/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-2/wal
14:00:01.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9e7e867b3f34e81b9c9d465e50ab395"
14:00:01.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:01 on dist-test-slave-vpvm"
14:00:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.856510 28405 fs_report.cc:389] FS layout report
14:00:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:00:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-2/wal
14:00:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-2/wal
14:00:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-2/data/data
14:00:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:00:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:00:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:00:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:00:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:00:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:00:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:00:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:00:01.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:00:01.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:00:01.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:00:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.908492 28292 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } attempt: 6
14:00:01.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:01.915726 28292 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.131.252:44291: Network error: Client connection negotiation failed: client connection to 127.27.131.252:44291: connect: Connection refused (error 111)
14:00:01.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.952674 28405 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:00:01.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.954552 28405 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:00:01.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:01.955050 28405 kserver.cc:163] Server-wide thread pool size limit: 3276
14:00:02.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.033002 28405 rpc_server.cc:307] RPC server started. Bound to: 127.27.131.252:44291
14:00:02.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.033069 28533 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.131.252:44291 every 8 connection(s)
14:00:02.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.035759 28405 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-2/data/info.pb
14:00:02.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.036509 28175 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu as pid 28405
14:00:02.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.037012 28175 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/master-2/wal/instance
14:00:02.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.045012 28535 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:00:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.063308 28535 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } has no permanent_uuid. Determining permanent_uuid...
14:00:02.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.087978 28535 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } has no permanent_uuid. Determining permanent_uuid...
14:00:02.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.099622 28535 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } has no permanent_uuid. Determining permanent_uuid...
14:00:02.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.116858 28535 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395: Bootstrap starting.
14:00:02.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.124368 28535 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395: Neither blocks nor log segments found. Creating new log.
14:00:02.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.126333 28535 log.cc:826] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395: Log is configured to *not* fsync() on all Append() calls
14:00:02.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.132014 28535 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395: No bootstrap required, opened a new log
14:00:02.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.152736 28535 raft_consensus.cc:359] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } }
14:00:02.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.153517 28535 raft_consensus.cc:385] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:00:02.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.153832 28535 raft_consensus.cc:740] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9e7e867b3f34e81b9c9d465e50ab395, State: Initialized, Role: FOLLOWER
14:00:02.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.155045 28535 consensus_queue.cc:260] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } }
14:00:02.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.157923 28547 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } } }
14:00:02.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.158690 28547 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [sys.catalog]: This master's current role is: FOLLOWER
14:00:02.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.159483 28535 sys_catalog.cc:565] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [sys.catalog]: configured and running, proceeding with master startup.
14:00:02.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:02.187842 28569 catalog_manager.cc:1594] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:02.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:02.188197 28569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:02.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.182816 28405 master_runner.cc:187] Error getting master registration for 127.27.131.254:44375: OK, instance_id { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" instance_seqno: 1786543198231903 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:00:02.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.208276 28405 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:00:02.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:02.211133 28460 tablet.cc:2406] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:00:02.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.332544 28547 raft_consensus.cc:493] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:00:02.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.332998 28547 raft_consensus.cc:515] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } }
14:00:02.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.335209 28547 leader_election.cc:290] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers dbbb07ceed49450188856ccbeb983ad0 (127.27.131.254:44375), 572befe5b131437d99e612431649b95b (127.27.131.253:45199)
14:00:02.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.424887 28403 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } attempt: 7
14:00:02.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.447788 28403 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b: Bootstrap starting.
14:00:02.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.456300 28403 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b: Neither blocks nor log segments found. Creating new log.
14:00:02.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.459007 28403 log.cc:826] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b: Log is configured to *not* fsync() on all Append() calls
14:00:02.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.464363 28403 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b: No bootstrap required, opened a new log
14:00:02.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.491631 28403 raft_consensus.cc:359] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } }
14:00:02.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.493163 28403 raft_consensus.cc:385] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:00:02.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.493499 28403 raft_consensus.cc:740] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 572befe5b131437d99e612431649b95b, State: Initialized, Role: FOLLOWER
14:00:02.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.494613 28403 consensus_queue.cc:260] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } }
14:00:02.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.498024 28575 sys_catalog.cc:455] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } } }
14:00:02.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.499308 28575 sys_catalog.cc:458] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b [sys.catalog]: This master's current role is: FOLLOWER
14:00:02.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.500047 28403 sys_catalog.cc:565] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b [sys.catalog]: configured and running, proceeding with master startup.
14:00:02.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.501495 28363 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "572befe5b131437d99e612431649b95b" is_pre_election: true
14:00:02.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.502458 28363 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d9e7e867b3f34e81b9c9d465e50ab395 in term 0.
14:00:02.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.506726 28453 leader_election.cc:304] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 572befe5b131437d99e612431649b95b, d9e7e867b3f34e81b9c9d465e50ab395; no voters:
14:00:02.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.507790 28547 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:00:02.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.508190 28547 raft_consensus.cc:493] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:00:02.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.508508 28547 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [term 0 FOLLOWER]: Advancing to term 1
14:00:02.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.516701 28547 raft_consensus.cc:515] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } }
14:00:02.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.520180 28547 leader_election.cc:290] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [CANDIDATE]: Term 1 election: Requested vote from peers dbbb07ceed49450188856ccbeb983ad0 (127.27.131.254:44375), 572befe5b131437d99e612431649b95b (127.27.131.253:45199)
14:00:02.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.525650 28363 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "572befe5b131437d99e612431649b95b"
14:00:02.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.527053 28363 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b [term 0 FOLLOWER]: Advancing to term 1
14:00:02.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.537788 28363 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d9e7e867b3f34e81b9c9d465e50ab395 in term 1.
14:00:02.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.539278 28453 leader_election.cc:304] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 572befe5b131437d99e612431649b95b, d9e7e867b3f34e81b9c9d465e50ab395; no voters:
14:00:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.540393 28547 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [term 1 FOLLOWER]: Leader election won for term 1
14:00:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.543040 28547 raft_consensus.cc:697] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [term 1 LEADER]: Becoming Leader. State: Replica: d9e7e867b3f34e81b9c9d465e50ab395, State: Running, Role: LEADER
14:00:02.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.544499 28547 consensus_queue.cc:237] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } }
14:00:02.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.566294 28583 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d9e7e867b3f34e81b9c9d465e50ab395. Latest consensus state: current_term: 1 leader_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } } }
14:00:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.568241 28583 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [sys.catalog]: This master's current role is: LEADER
14:00:02.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.574967 28593 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
14:00:02.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.584045 28593 catalog_manager.cc:1520] Initializing Kudu cluster ID...
14:00:02.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:02.610399 28595 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:02.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:02.618086 28595 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:02.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.618386 28363 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b [term 1 FOLLOWER]: Refusing update from remote peer d9e7e867b3f34e81b9c9d465e50ab395: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:00:02.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.620345 28583 consensus_queue.cc:1048] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [LEADER]: Connected to new peer: Peer: permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:00:02.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.595345 28294 master_runner.cc:187] Error getting master registration for 127.27.131.254:44375: OK, instance_id { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" instance_seqno: 1786543198231903 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:00:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.660148 28294 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:00:02.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.673017 28575 sys_catalog.cc:455] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b [sys.catalog]: SysCatalogTable state changed. Reason: New leader d9e7e867b3f34e81b9c9d465e50ab395. Latest consensus state: current_term: 1 leader_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } } }
14:00:02.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.679283 28596 mvcc.cc:204] Tried to move back new op lower bound from 7317680957907521536 to 7317680957635555328. Current Snapshot: MvccSnapshot[applied={T|T < 7317680957907521536}]
14:00:02.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.681015 28575 sys_catalog.cc:458] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b [sys.catalog]: This master's current role is: FOLLOWER
14:00:02.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.693687 28547 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } } }
14:00:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.694496 28547 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [sys.catalog]: This master's current role is: LEADER
14:00:02.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.698133 28593 catalog_manager.cc:1383] Generated new cluster ID: fa63aa03c59f41269c0609e69b204dda
14:00:02.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.698447 28593 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
14:00:02.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.703202 28575 sys_catalog.cc:455] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } } }
14:00:02.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.703933 28575 sys_catalog.cc:458] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b [sys.catalog]: This master's current role is: FOLLOWER
14:00:02.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.740008 28593 catalog_manager.cc:1406] Generated new certificate authority record
14:00:02.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.742226 28593 catalog_manager.cc:1540] Loading token signing keys...
14:00:02.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.788161 28593 catalog_manager.cc:6092] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395: Generated new TSK 0
14:00:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.789394 28593 catalog_manager.cc:1550] Initializing in-progress tserver states...
14:00:02.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.819936 28175 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:02.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:02.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-0/wal
14:00:02.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-0/data
14:00:02.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:00:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:00:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:00:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-0/logs
14:00:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-0/data/info.pb
14:00:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:00:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:00:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:00:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:00:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:00:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:00:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.131.193:0
14:00:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.131.193
14:00:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.131.193
14:00:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.131.254:44375,127.27.131.253:45199,127.27.131.252:44291
14:00:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.131.212:43191
14:00:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:00:02.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.949105 28292 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } attempt: 7
14:00:02.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:02.996234 28292 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0: Bootstrap starting.
14:00:03.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.004954 28292 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0: Neither blocks nor log segments found. Creating new log.
14:00:03.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.008250 28292 log.cc:826] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0: Log is configured to *not* fsync() on all Append() calls
14:00:03.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.024981 28292 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0: No bootstrap required, opened a new log
14:00:03.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.224876 28292 raft_consensus.cc:359] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } }
14:00:03.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.234123 28292 raft_consensus.cc:385] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:00:03.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.238111 28292 raft_consensus.cc:740] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dbbb07ceed49450188856ccbeb983ad0, State: Initialized, Role: FOLLOWER
14:00:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.239662 28292 consensus_queue.cc:260] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } }
14:00:03.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.255470 28611 sys_catalog.cc:455] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } } }
14:00:03.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.269730 28611 sys_catalog.cc:458] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0 [sys.catalog]: This master's current role is: FOLLOWER
14:00:03.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.259626 28257 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dbbb07ceed49450188856ccbeb983ad0" is_pre_election: true
14:00:03.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.262470 28255 raft_consensus.cc:3060] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0 [term 0 FOLLOWER]: Advancing to term 1
14:00:03.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.262562 28256 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dbbb07ceed49450188856ccbeb983ad0"
14:00:03.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.259430 28292 sys_catalog.cc:565] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0 [sys.catalog]: configured and running, proceeding with master startup.
14:00:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.281426 28255 raft_consensus.cc:1275] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0 [term 1 FOLLOWER]: Refusing update from remote peer d9e7e867b3f34e81b9c9d465e50ab395: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:00:03.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.283542 28583 consensus_queue.cc:1048] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:00:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.365253 28611 sys_catalog.cc:455] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d9e7e867b3f34e81b9c9d465e50ab395. Latest consensus state: current_term: 1 leader_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } } }
14:00:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.365355 28625 sys_catalog.cc:455] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } } }
14:00:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.366050 28611 sys_catalog.cc:458] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0 [sys.catalog]: This master's current role is: FOLLOWER
14:00:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.366210 28625 sys_catalog.cc:458] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0 [sys.catalog]: This master's current role is: FOLLOWER
14:00:03.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.401011 28643 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:00:03.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:03.478408 28606 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:00:03.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:03.479084 28606 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:00:03.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:03.479909 28606 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:00:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:03.544137 28606 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:00:03.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:03.545513 28606 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.131.193
14:00:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.612324 28606 tablet_server_runner.cc:78] Tablet server non-default flags:
14:00:03.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-0/logs
14:00:03.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:03.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:00:03.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:03.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.131.212:43191
14:00:03.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:03.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:03.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-0/data
14:00:03.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-0/wal
14:00:03.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:03.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:03.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.131.193:0
14:00:03.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:00:03.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:03.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:03.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-0/data/info.pb
14:00:03.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.131.193
14:00:03.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:03.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.131.254:44375,127.27.131.253:45199,127.27.131.252:44291
14:00:03.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:00:03.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:03.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:00:03.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:00:03.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:03.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.131.193
14:00:03.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:00:03.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:00:03.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:00:03.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 70c51312f1186a8ae9f46647160b7730123f7601
14:00:03.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:00:03.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 13:43:22 UTC on e7f111948823
14:00:03.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13307
14:00:03.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:00:03.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.614121 28606 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:00:03.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.616416 28606 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:00:03.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.626151 28595 catalog_manager.cc:1295] Loaded cluster ID: fa63aa03c59f41269c0609e69b204dda
14:00:03.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.626425 28595 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b: loading cluster ID for follower catalog manager: success
14:00:03.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.634509 28595 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b: acquiring CA information for follower catalog manager: success
14:00:03.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.641877 28595 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 572befe5b131437d99e612431649b95b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:00:03.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:03.659736 28650 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:03.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.752277 28660 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dbbb07ceed49450188856ccbeb983ad0" member_type: VOTER last_known_addr { host: "127.27.131.254" port: 44375 } } peers { permanent_uuid: "572befe5b131437d99e612431649b95b" member_type: VOTER last_known_addr { host: "127.27.131.253" port: 45199 } } peers { permanent_uuid: "d9e7e867b3f34e81b9c9d465e50ab395" member_type: VOTER last_known_addr { host: "127.27.131.252" port: 44291 } } }
14:00:03.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:03.756691 28660 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9e7e867b3f34e81b9c9d465e50ab395 [sys.catalog]: This master's current role is: LEADER
14:00:04.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:04.419024 28643 catalog_manager.cc:1295] Loaded cluster ID: fa63aa03c59f41269c0609e69b204dda
14:00:04.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:04.419337 28643 catalog_manager.cc:1588] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0: loading cluster ID for follower catalog manager: success
14:00:04.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:04.439209 28643 catalog_manager.cc:1610] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0: acquiring CA information for follower catalog manager: success
14:00:04.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:04.446228 28643 catalog_manager.cc:1638] T 00000000000000000000000000000000 P dbbb07ceed49450188856ccbeb983ad0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:00:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:05.440064 28657 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:05.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:03.663266 28651 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:05.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.446806 28606 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:00:05.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:05.446679 28656 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1783 milliseconds
14:00:05.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.449419 28606 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:00:05.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.454063 28606 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:00:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.455523 28606 hybrid_clock.cc:648] HybridClock initialized: now 1786543205455475 us; error 70 us; skew 500 ppm
14:00:05.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.456882 28606 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:00:05.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.465454 28606 webserver.cc:533] Webserver started at http://127.27.131.193:42495/ using document root <none> and password file <none>
14:00:05.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.466804 28606 fs_manager.cc:362] Metadata directory not provided
14:00:05.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.467092 28606 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:00:05.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.467682 28606 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:00:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.474344 28606 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-0/data/instance:
14:00:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0155151757c544ab8d6c7bfd5057c839"
14:00:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:05 on dist-test-slave-vpvm"
14:00:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.476011 28606 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-0/wal/instance:
14:00:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0155151757c544ab8d6c7bfd5057c839"
14:00:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:05 on dist-test-slave-vpvm"
14:00:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.487645 28606 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.012s sys 0.000s
14:00:05.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.496047 28670 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:00:05.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.497571 28606 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.001s
14:00:05.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.498020 28606 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-0/data,/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-0/wal
14:00:05.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0155151757c544ab8d6c7bfd5057c839"
14:00:05.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:05 on dist-test-slave-vpvm"
14:00:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.498471 28606 fs_report.cc:389] FS layout report
14:00:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:00:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-0/wal
14:00:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-0/wal
14:00:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-0/data/data
14:00:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:00:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:00:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:00:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:00:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:00:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:00:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:00:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:00:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:00:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:00:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:00:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.574463 28606 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:00:05.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.576619 28606 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:00:05.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.577210 28606 kserver.cc:163] Server-wide thread pool size limit: 3276
14:00:05.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.580817 28606 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:00:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.586944 28606 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:00:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.587241 28606 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:00:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.587558 28606 ts_tablet_manager.cc:616] Registered 0 tablets
14:00:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.587776 28606 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:00:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.772235 28606 rpc_server.cc:307] RPC server started. Bound to: 127.27.131.193:34541
14:00:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.772313 28842 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.131.193:34541 every 8 connection(s)
14:00:05.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.776211 28606 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-0/data/info.pb
14:00:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.781433 28175 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu as pid 28606
14:00:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.781934 28175 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-0/wal/instance
14:00:05.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.792814 28175 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:05.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:05.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-1/wal
14:00:05.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-1/data
14:00:05.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:00:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:00:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:00:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-1/logs
14:00:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-1/data/info.pb
14:00:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:00:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:00:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:00:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:00:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:00:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:00:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.131.194:0
14:00:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.131.194
14:00:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.131.194
14:00:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.131.254:44375,127.27.131.253:45199,127.27.131.252:44291
14:00:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.131.212:43191
14:00:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:00:05.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.856734 28843 heartbeater.cc:344] Connected to a master server at 127.27.131.253:45199
14:00:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.857293 28843 heartbeater.cc:461] Registering TS with master...
14:00:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.858968 28843 heartbeater.cc:507] Master 127.27.131.253:45199 requested a full tablet report, sending...
14:00:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.859082 28853 heartbeater.cc:344] Connected to a master server at 127.27.131.254:44375
14:00:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.859457 28853 heartbeater.cc:461] Registering TS with master...
14:00:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.860407 28853 heartbeater.cc:507] Master 127.27.131.254:44375 requested a full tablet report, sending...
14:00:05.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.862633 28844 heartbeater.cc:344] Connected to a master server at 127.27.131.252:44291
14:00:05.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.862985 28844 heartbeater.cc:461] Registering TS with master...
14:00:05.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.863917 28844 heartbeater.cc:507] Master 127.27.131.252:44291 requested a full tablet report, sending...
14:00:05.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.864861 28241 ts_manager.cc:194] Registered new tserver with Master: 0155151757c544ab8d6c7bfd5057c839 (127.27.131.193:34541)
14:00:05.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.868314 28477 ts_manager.cc:194] Registered new tserver with Master: 0155151757c544ab8d6c7bfd5057c839 (127.27.131.193:34541)
14:00:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.872058 28477 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.27.131.193:34181
14:00:05.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:05.872470 28352 ts_manager.cc:194] Registered new tserver with Master: 0155151757c544ab8d6c7bfd5057c839 (127.27.131.193:34541)
14:00:06.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:06.210186 28852 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:00:06.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:06.210745 28852 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:00:06.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:06.211186 28852 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:00:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:06.248559 28852 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:00:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:06.250533 28852 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.131.194
14:00:06.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:06.299489 28852 tablet_server_runner.cc:78] Tablet server non-default flags:
14:00:06.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-1/logs
14:00:06.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:06.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:00:06.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:06.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.131.212:43191
14:00:06.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-1/data
14:00:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-1/wal
14:00:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.131.194:0
14:00:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:00:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-1/data/info.pb
14:00:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.131.194
14:00:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.131.254:44375,127.27.131.253:45199,127.27.131.252:44291
14:00:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:00:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:00:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:00:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.131.194
14:00:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:00:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:00:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:00:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 70c51312f1186a8ae9f46647160b7730123f7601
14:00:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:00:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 13:43:22 UTC on e7f111948823
14:00:06.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13307
14:00:06.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:00:06.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:06.301451 28852 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:00:06.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:06.304073 28852 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:00:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:06.321976 28866 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:06.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:06.876560 28844 heartbeater.cc:499] Master 127.27.131.252:44291 was elected leader, sending a full tablet report...
14:00:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:07.721849 28864 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee40 after lost signal to thread 28852
14:00:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:07.892964 28864 kernel_stack_watchdog.cc:198] Thread 28852 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 394ms:
14:00:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:00:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:00:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:00:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:00:07.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:06.328174 28868 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:07.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:07.893456 28852 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.569s user 0.596s sys 0.955s
14:00:07.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:07.894793 28870 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1568 milliseconds
14:00:07.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:07.895306 28852 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.571s user 0.596s sys 0.956s
14:00:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:07.896057 28852 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:00:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:07.896130 28871 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:07.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:07.900791 28852 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:00:07.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:07.903148 28852 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:00:07.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:07.904583 28852 hybrid_clock.cc:648] HybridClock initialized: now 1786543207904503 us; error 95 us; skew 500 ppm
14:00:07.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:07.905632 28852 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:00:07.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:07.912829 28852 webserver.cc:533] Webserver started at http://127.27.131.194:35621/ using document root <none> and password file <none>
14:00:07.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:07.913941 28852 fs_manager.cc:362] Metadata directory not provided
14:00:07.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:07.914234 28852 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:00:07.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:07.914829 28852 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:00:07.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:07.919512 28852 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-1/data/instance:
14:00:07.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7efd62a493414036a72e3b374862900c"
14:00:07.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:07 on dist-test-slave-vpvm"
14:00:07.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:07.920974 28852 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-1/wal/instance:
14:00:07.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7efd62a493414036a72e3b374862900c"
14:00:07.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:07 on dist-test-slave-vpvm"
14:00:07.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:07.928844 28852 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.001s
14:00:07.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:07.934995 28884 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:00:07.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:07.936268 28852 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
14:00:07.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:07.936674 28852 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-1/data,/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-1/wal
14:00:07.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7efd62a493414036a72e3b374862900c"
14:00:07.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:07 on dist-test-slave-vpvm"
14:00:07.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:07.937073 28852 fs_report.cc:389] FS layout report
14:00:07.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:00:07.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-1/wal
14:00:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-1/wal
14:00:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-1/data/data
14:00:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:00:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:00:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:00:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:00:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:00:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:00:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:00:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:00:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:00:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:00:07.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:00:08.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.019618 28852 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:00:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.021245 28852 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:00:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.021781 28852 kserver.cc:163] Server-wide thread pool size limit: 3276
14:00:08.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.024880 28852 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:00:08.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.029562 28852 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:00:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.029793 28852 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:00:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.030098 28852 ts_tablet_manager.cc:616] Registered 0 tablets
14:00:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.030318 28852 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:00:08.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.193463 28852 rpc_server.cc:307] RPC server started. Bound to: 127.27.131.194:37783
14:00:08.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.193590 29051 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.131.194:37783 every 8 connection(s)
14:00:08.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.196190 28852 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-1/data/info.pb
14:00:08.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.205926 28175 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu as pid 28852
14:00:08.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.206722 28175 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-1/wal/instance
14:00:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.230470 28175 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-2/wal
14:00:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-2/data
14:00:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:00:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:00:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:00:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-2/logs
14:00:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-2/data/info.pb
14:00:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:00:08.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:00:08.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:00:08.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:00:08.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:08.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:08.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:08.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:00:08.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:00:08.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.131.195:0
14:00:08.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.131.195
14:00:08.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.131.195
14:00:08.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:08.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.131.254:44375,127.27.131.253:45199,127.27.131.252:44291
14:00:08.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.131.212:43191
14:00:08.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:08.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:08.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:00:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.250324 29054 heartbeater.cc:344] Connected to a master server at 127.27.131.253:45199
14:00:08.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.251755 29054 heartbeater.cc:461] Registering TS with master...
14:00:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.254380 29054 heartbeater.cc:507] Master 127.27.131.253:45199 requested a full tablet report, sending...
14:00:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.257977 29055 heartbeater.cc:344] Connected to a master server at 127.27.131.252:44291
14:00:08.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.258307 29055 heartbeater.cc:461] Registering TS with master...
14:00:08.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.258546 28352 ts_manager.cc:194] Registered new tserver with Master: 7efd62a493414036a72e3b374862900c (127.27.131.194:37783)
14:00:08.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.259232 29056 heartbeater.cc:344] Connected to a master server at 127.27.131.254:44375
14:00:08.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.259315 29055 heartbeater.cc:507] Master 127.27.131.252:44291 requested a full tablet report, sending...
14:00:08.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.259613 29056 heartbeater.cc:461] Registering TS with master...
14:00:08.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.260566 29056 heartbeater.cc:507] Master 127.27.131.254:44375 requested a full tablet report, sending...
14:00:08.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.263044 28477 ts_manager.cc:194] Registered new tserver with Master: 7efd62a493414036a72e3b374862900c (127.27.131.194:37783)
14:00:08.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.264423 28477 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.27.131.194:54197
14:00:08.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.266726 28241 ts_manager.cc:194] Registered new tserver with Master: 7efd62a493414036a72e3b374862900c (127.27.131.194:37783)
14:00:08.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:08.587443 29066 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:00:08.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:08.587953 29066 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:00:08.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:08.588470 29066 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:00:08.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:08.624500 29066 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:00:08.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:08.625659 29066 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.131.195
14:00:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.665302 29066 tablet_server_runner.cc:78] Tablet server non-default flags:
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-2/logs
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.131.212:43191
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-2/data
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-2/wal
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.131.195:0
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-2/data/info.pb
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.131.195
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.131.254:44375,127.27.131.253:45199,127.27.131.252:44291
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:00:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:00:08.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:08.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.131.195
14:00:08.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:00:08.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:00:08.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:00:08.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 70c51312f1186a8ae9f46647160b7730123f7601
14:00:08.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:00:08.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 13:43:22 UTC on e7f111948823
14:00:08.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13307
14:00:08.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:00:08.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.666898 29066 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:00:08.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:08.668658 29066 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:00:08.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:08.682256 29079 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:09.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.268452 29055 heartbeater.cc:499] Master 127.27.131.252:44291 was elected leader, sending a full tablet report...
14:00:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:08.683048 29080 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:09.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:08.687623 29082 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:09.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:09.791229 29081 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
14:00:09.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.791690 29066 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:00:09.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.795817 29066 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:00:09.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.798621 29066 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:00:09.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.800132 29066 hybrid_clock.cc:648] HybridClock initialized: now 1786543209800067 us; error 71 us; skew 500 ppm
14:00:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.801008 29066 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:00:09.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.807440 29066 webserver.cc:533] Webserver started at http://127.27.131.195:37101/ using document root <none> and password file <none>
14:00:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.808406 29066 fs_manager.cc:362] Metadata directory not provided
14:00:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.808615 29066 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:00:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.809017 29066 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:00:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.813452 29066 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-2/data/instance:
14:00:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad9085c38fd94cc19707c55b51fa286e"
14:00:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:09 on dist-test-slave-vpvm"
14:00:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.814664 29066 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-2/wal/instance:
14:00:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad9085c38fd94cc19707c55b51fa286e"
14:00:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:09 on dist-test-slave-vpvm"
14:00:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.822618 29066 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
14:00:09.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.828831 29095 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:00:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.830120 29066 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
14:00:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.830529 29066 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-2/data,/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-2/wal
14:00:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad9085c38fd94cc19707c55b51fa286e"
14:00:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:09 on dist-test-slave-vpvm"
14:00:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.830868 29066 fs_report.cc:389] FS layout report
14:00:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:00:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-2/wal
14:00:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-2/wal
14:00:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-2/data/data
14:00:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:00:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:00:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:00:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:00:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:00:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:00:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:00:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:00:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:00:09.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:00:09.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:00:09.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.885535 29066 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:00:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.887331 29066 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:00:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.887777 29066 kserver.cc:163] Server-wide thread pool size limit: 3276
14:00:09.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.890738 29066 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:00:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.896430 29066 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:00:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.896663 29066 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:00:09.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.896898 29066 ts_tablet_manager.cc:616] Registered 0 tablets
14:00:09.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:09.897073 29066 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:00:10.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.171407 29066 rpc_server.cc:307] RPC server started. Bound to: 127.27.131.195:45119
14:00:10.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.171674 29278 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.131.195:45119 every 8 connection(s)
14:00:10.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.174427 29066 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-2/data/info.pb
14:00:10.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.183554 28175 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu as pid 29066
14:00:10.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.184087 28175 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster10469899720837266292/ts-2/wal/instance
14:00:10.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.265364 29281 heartbeater.cc:344] Connected to a master server at 127.27.131.252:44291
14:00:10.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.266065 29281 heartbeater.cc:461] Registering TS with master...
14:00:10.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.267787 29281 heartbeater.cc:507] Master 127.27.131.252:44291 requested a full tablet report, sending...
14:00:10.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.276443 29284 heartbeater.cc:344] Connected to a master server at 127.27.131.254:44375
14:00:10.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.276865 29284 heartbeater.cc:461] Registering TS with master...
14:00:10.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.277824 29279 heartbeater.cc:344] Connected to a master server at 127.27.131.253:45199
14:00:10.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.278211 29279 heartbeater.cc:461] Registering TS with master...
14:00:10.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.279948 29284 heartbeater.cc:507] Master 127.27.131.254:44375 requested a full tablet report, sending...
14:00:10.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.281260 29279 heartbeater.cc:507] Master 127.27.131.253:45199 requested a full tablet report, sending...
14:00:10.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.281313 28476 ts_manager.cc:194] Registered new tserver with Master: ad9085c38fd94cc19707c55b51fa286e (127.27.131.195:45119)
14:00:10.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.284119 28476 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.27.131.195:48395
14:00:10.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.284411 28352 ts_manager.cc:194] Registered new tserver with Master: ad9085c38fd94cc19707c55b51fa286e (127.27.131.195:45119)
14:00:10.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.288164 28241 ts_manager.cc:194] Registered new tserver with Master: ad9085c38fd94cc19707c55b51fa286e (127.27.131.195:45119)
14:00:10.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:10.304066 28175 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
14:00:10.326 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:00:10.327 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:00:10.337 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.27.131.254"
port: 44375
}
bound_http_address {
host: "127.27.131.254"
port: 33303
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.27.131.253"
port: 45199
}
bound_http_address {
host: "127.27.131.253"
port: 34475
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.27.131.252"
port: 44291
}
bound_http_address {
host: "127.27.131.252"
port: 35473
}
}
}
14:00:10.341 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:00:10.343 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.27.131.193"
port: 34541
}
bound_http_address {
host: "127.27.131.193"
port: 42495
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.27.131.194"
port: 37783
}
bound_http_address {
host: "127.27.131.194"
port: 35621
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.27.131.195"
port: 45119
}
bound_http_address {
host: "127.27.131.195"
port: 37101
}
}
}
14:00:10.344 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:00:10.561 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
14:00:10.628 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:00:10.722 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskTlaXut/test-tmp
14:00:10.722 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:00:10.723 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskTlaXut/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:00:10.727 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
14:00:11.098 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:00:11.099 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:00:11.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:11.102489 29300 mini_oidc.cc:150] Starting JWKS server
14:00:11.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:11.105726 29300 webserver.cc:533] Webserver started at http://127.0.0.1:36509/ using document root <none> and password file <none>
14:00:11.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:11.106208 29300 mini_oidc.cc:186] Starting OIDC Discovery server
14:00:11.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:11.108155 29300 webserver.cc:533] Webserver started at http://127.0.0.1:39149/ using document root <none> and password file <none>
14:00:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
14:00:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/principal' for realm 'KRBTEST.COM',
14:00:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
14:00:11.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:11 dist-test-slave-vpvm krb5kdc[29316](info): setting up network...
14:00:11.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
14:00:11.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:11 dist-test-slave-vpvm krb5kdc[29316](info): set up 2 sockets
14:00:11.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:11 dist-test-slave-vpvm krb5kdc[29316](info): commencing operation
14:00:11.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
14:00:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:11.292249 29281 heartbeater.cc:499] Master 127.27.131.252:44291 was elected leader, sending a full tablet report...
14:00:13.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:13.312084 29300 mini_kdc.cc:121] Time spent starting KDC: real 2.168s user 0.006s sys 0.025s
14:00:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
14:00:13.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:00:13.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
14:00:13.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
14:00:13.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:00:13.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
14:00:13.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
14:00:13.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:00:13.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
14:00:13.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:00:13.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/test-user.keytab.
14:00:13.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/test-user.keytab.
14:00:13.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:13 dist-test-slave-vpvm krb5kdc[29316](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543213, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:13.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:00:13.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T14:00:13Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:00:13.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T14:00:13Z Disabled control of system clock
14:00:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.157.62@KRBTEST.COM; defaulting to no policy
14:00:13.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:13.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.157.62@KRBTEST.COM" created.
14:00:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.157.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:13.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.157.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:13.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.157.62@KRBTEST.COM; defaulting to no policy
14:00:13.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:13.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.157.62@KRBTEST.COM" created.
14:00:13.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:13.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.157.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:13.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.157.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:13.594897 29300 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/wal
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/data
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/logs
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/data/info.pb
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:00:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.157.62:34669
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.157.62
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.157.62
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.157.20:43703
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.157.62:34669,127.28.157.61:36377,127.28.157.60:35027
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36509/jwks.json
14:00:13.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:00:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:00:13.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:13.931533 29349 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:00:13.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:13.932303 29349 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:00:13.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:13.933063 29349 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:00:13.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:13.966991 29349 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:00:13.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:13.967283 29349 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:00:13.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:13.967708 29349 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:00:13.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:13.968015 29349 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:00:13.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:13.968320 29349 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:00:13.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:13.968629 29349 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:00:13.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:13.968851 29349 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36509/jwks.json
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:14.010460 29349 master_runner.cc:387] Master server non-default flags:
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/logs
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.157.20:43703
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/data
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/wal
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.157.62:34669,127.28.157.61:36377,127.28.157.60:35027
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.157.62
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.157.62:34669
14:00:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:00:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:00:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:00:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36509/jwks.json
14:00:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:00:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:00:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:00:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/data/info.pb
14:00:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.157.62
14:00:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:00:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:00:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:00:14.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:14.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:00:14.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:00:14.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:00:14.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 70c51312f1186a8ae9f46647160b7730123f7601
14:00:14.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:00:14.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 13:43:22 UTC on e7f111948823
14:00:14.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13307
14:00:14.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:00:14.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:14.012369 29349 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:00:14.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:14.013849 29349 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:00:14.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:14.035202 29358 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:15.443737 29355 debug-util.cc:398] Leaking SignalData structure 0x7b0800034fe0 after lost signal to thread 29349
14:00:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:15.642102 29349 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.595s user 0.582s sys 0.974s
14:00:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:14.046849 29359 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:15.642942 29349 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.596s user 0.582s sys 0.974s
14:00:15.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:15.647385 29363 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:15.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:15.648490 29361 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1596 milliseconds
14:00:15.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.648523 29349 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:00:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.650000 29349 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:00:15.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.653083 29349 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:00:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.654488 29349 hybrid_clock.cc:648] HybridClock initialized: now 1786543215654465 us; error 85 us; skew 500 ppm
14:00:15.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.655462 29349 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:00:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:15 dist-test-slave-vpvm krb5kdc[29316](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543215, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:15.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.664808 29349 init.cc:377] Logged in from keytab as kudu/127.28.157.62@KRBTEST.COM (short username kudu)
14:00:15.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.676296 29349 webserver.cc:533] Webserver started at http://127.28.157.62:36765/ using document root <none> and password file <none>
14:00:15.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.677520 29349 fs_manager.cc:362] Metadata directory not provided
14:00:15.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.677760 29349 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:00:15.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.678301 29349 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:00:15.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.683627 29349 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/data/instance:
14:00:15.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "881c1b6802e24f6d9f3fb8ca5020114c"
14:00:15.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:15 on dist-test-slave-vpvm"
14:00:15.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.684860 29349 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/wal/instance:
14:00:15.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "881c1b6802e24f6d9f3fb8ca5020114c"
14:00:15.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:15 on dist-test-slave-vpvm"
14:00:15.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.695926 29349 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.005s sys 0.007s
14:00:15.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.704085 29381 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:00:15.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.705652 29349 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.002s sys 0.004s
14:00:15.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.706089 29349 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/data,/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/wal
14:00:15.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "881c1b6802e24f6d9f3fb8ca5020114c"
14:00:15.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:15 on dist-test-slave-vpvm"
14:00:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.706513 29349 fs_report.cc:389] FS layout report
14:00:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:00:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/wal
14:00:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/wal
14:00:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/data/data
14:00:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:00:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:00:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:00:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:00:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:00:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:00:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:00:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:00:15.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:00:15.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:00:15.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:00:15.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.781615 29349 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:00:15.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.792328 29349 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:00:15.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:15.792898 29349 kserver.cc:163] Server-wide thread pool size limit: 3276
14:00:16.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:16.001680 29349 rpc_server.cc:307] RPC server started. Bound to: 127.28.157.62:34669
14:00:16.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:16.002226 29462 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.157.62:34669 every 8 connection(s)
14:00:16.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:16.006117 29349 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/data/info.pb
14:00:16.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:16.015359 29300 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu as pid 29349
14:00:16.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:16.016005 29300 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/wal/instance
14:00:16.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:16.018883 29463 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:00:16.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.157.61@KRBTEST.COM; defaulting to no policy
14:00:16.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:16.043084 29463 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } has no permanent_uuid. Determining permanent_uuid...
14:00:16.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.157.61@KRBTEST.COM" created.
14:00:16.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:16.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.157.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:16.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.157.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:16.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:16 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543215, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.62@KRBTEST.COM for kudu/127.28.157.62@KRBTEST.COM
14:00:16.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.157.61@KRBTEST.COM; defaulting to no policy
14:00:16.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:16.102694 29463 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } has no permanent_uuid. Determining permanent_uuid...
14:00:16.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:16.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.157.61@KRBTEST.COM" created.
14:00:16.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.105377 29382 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.157.61:36377: connect: Connection refused (error 111)
14:00:16.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.110244 29463 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.61:36377: Network error: Client connection negotiation failed: client connection to 127.28.157.61:36377: connect: Connection refused (error 111)
14:00:16.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:16.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.157.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:16.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.157.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:16.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:16.132674 29300 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:16.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:16.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/wal
14:00:16.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/data
14:00:16.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:00:16.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:00:16.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:00:16.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:16.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:16.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/logs
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/data/info.pb
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.157.61:36377
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.157.61
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.157.61
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.157.20:43703
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.157.62:34669,127.28.157.61:36377,127.28.157.60:35027
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36509/jwks.json
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:00:16.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:16.159790 29463 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } attempt: 1
14:00:16.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.164538 29463 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.61:36377: Network error: Client connection negotiation failed: client connection to 127.28.157.61:36377: connect: Connection refused (error 111)
14:00:16.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:16.233021 29463 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } attempt: 2
14:00:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.238432 29463 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.61:36377: Network error: Client connection negotiation failed: client connection to 127.28.157.61:36377: connect: Connection refused (error 111)
14:00:16.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:16.329874 29463 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } attempt: 3
14:00:16.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.334858 29463 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.61:36377: Network error: Client connection negotiation failed: client connection to 127.28.157.61:36377: connect: Connection refused (error 111)
14:00:16.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:16.478245 29463 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } attempt: 4
14:00:16.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.483186 29463 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.61:36377: Network error: Client connection negotiation failed: client connection to 127.28.157.61:36377: connect: Connection refused (error 111)
14:00:16.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.561638 29475 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:00:16.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.562340 29475 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:00:16.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.562938 29475 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:00:16.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.597045 29475 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:00:16.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.597290 29475 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:00:16.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.597569 29475 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:00:16.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.597807 29475 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:00:16.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.598064 29475 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:00:16.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.598274 29475 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:00:16.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.598431 29475 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36509/jwks.json
14:00:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:16.638026 29475 master_runner.cc:387] Master server non-default flags:
14:00:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/logs
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.157.20:43703
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/data
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/wal
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.157.62:34669,127.28.157.61:36377,127.28.157.60:35027
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.157.61
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.157.61:36377
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36509/jwks.json
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/data/info.pb
14:00:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.157.61
14:00:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:00:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:00:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:00:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:00:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:00:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:00:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 70c51312f1186a8ae9f46647160b7730123f7601
14:00:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:00:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 13:43:22 UTC on e7f111948823
14:00:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13307
14:00:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:00:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:16.639334 29475 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:00:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:16.640339 29475 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:00:16.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.657637 29489 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:16.782717 29463 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } attempt: 5
14:00:16.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.793171 29463 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.61:36377: Network error: Client connection negotiation failed: client connection to 127.28.157.61:36377: connect: Connection refused (error 111)
14:00:17.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:17.340713 29463 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } attempt: 6
14:00:17.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:17.350906 29463 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.61:36377: Network error: Client connection negotiation failed: client connection to 127.28.157.61:36377: connect: Connection refused (error 111)
14:00:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:16.660392 29490 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:17.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:17.945526 29492 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:17.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:17.948952 29475 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.287s user 0.003s sys 0.014s
14:00:17.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:17.949244 29475 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.287s user 0.003s sys 0.014s
14:00:17.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:17.949443 29475 server_base.cc:1061] running on GCE node
14:00:17.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:17.950533 29475 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:00:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:17.953465 29475 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:00:17.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:17.955255 29475 hybrid_clock.cc:648] HybridClock initialized: now 1786543217955238 us; error 129 us; skew 500 ppm
14:00:17.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:17.956370 29475 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:00:17.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:17 dist-test-slave-vpvm krb5kdc[29316](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543217, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:17.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:17.967144 29475 init.cc:377] Logged in from keytab as kudu/127.28.157.61@KRBTEST.COM (short username kudu)
14:00:17.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:17.976385 29475 webserver.cc:533] Webserver started at http://127.28.157.61:39947/ using document root <none> and password file <none>
14:00:17.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:17.977415 29475 fs_manager.cc:362] Metadata directory not provided
14:00:17.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:17.977648 29475 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:00:17.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:17.978125 29475 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:00:17.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:17.983417 29475 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/data/instance:
14:00:17.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bab45e7c8bcd49f69477f23f98db2abf"
14:00:17.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:17 on dist-test-slave-vpvm"
14:00:17.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:17.984743 29475 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/wal/instance:
14:00:17.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bab45e7c8bcd49f69477f23f98db2abf"
14:00:17.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:17 on dist-test-slave-vpvm"
14:00:17.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:17.996716 29475 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.007s sys 0.005s
14:00:18.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.006599 29520 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:00:18.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.009104 29475 fs_manager.cc:730] Time spent opening block manager: real 0.007s user 0.000s sys 0.003s
14:00:18.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.009577 29475 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/data,/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/wal
14:00:18.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bab45e7c8bcd49f69477f23f98db2abf"
14:00:18.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:17 on dist-test-slave-vpvm"
14:00:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.010468 29475 fs_report.cc:389] FS layout report
14:00:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:00:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/wal
14:00:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/wal
14:00:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/data/data
14:00:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:00:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:00:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:00:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:00:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:00:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:00:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:00:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:00:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:00:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:00:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:00:18.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.103323 29475 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:00:18.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.109668 29475 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:00:18.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.110265 29475 kserver.cc:163] Server-wide thread pool size limit: 3276
14:00:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.256548 29475 rpc_server.cc:307] RPC server started. Bound to: 127.28.157.61:36377
14:00:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.260100 29602 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.157.61:36377 every 8 connection(s)
14:00:18.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.262643 29475 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/data/info.pb
14:00:18.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.271481 29300 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu as pid 29475
14:00:18.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.271976 29300 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/wal/instance
14:00:18.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.273679 29603 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:00:18.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.288753 29603 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } has no permanent_uuid. Determining permanent_uuid...
14:00:18.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.157.60@KRBTEST.COM; defaulting to no policy
14:00:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.157.60@KRBTEST.COM" created.
14:00:18.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:18 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543217, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.61@KRBTEST.COM for kudu/127.28.157.62@KRBTEST.COM
14:00:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.332361 29603 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } has no permanent_uuid. Determining permanent_uuid...
14:00:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.157.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.157.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:18.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:18 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543217, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.61@KRBTEST.COM for kudu/127.28.157.61@KRBTEST.COM
14:00:18.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.157.60@KRBTEST.COM; defaulting to no policy
14:00:18.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:18.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.157.60@KRBTEST.COM" created.
14:00:18.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.368117 29603 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } has no permanent_uuid. Determining permanent_uuid...
14:00:18.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.370927 29527 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:18.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.377075 29603 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.60:35027: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:18.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:18.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.157.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:18.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.157.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.396433 29300 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/wal
14:00:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/data
14:00:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:00:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:00:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:00:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/logs
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/data/info.pb
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.157.60:35027
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.157.60
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.157.60
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.157.20:43703
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.157.62:34669,127.28.157.61:36377,127.28.157.60:35027
14:00:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:00:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36509/jwks.json
14:00:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:00:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:00:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.411497 29463 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } attempt: 7
14:00:18.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:18 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543215, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.62@KRBTEST.COM for kudu/127.28.157.61@KRBTEST.COM
14:00:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.426608 29603 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } attempt: 1
14:00:18.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.431480 29603 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.60:35027: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:18.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.435462 29463 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } has no permanent_uuid. Determining permanent_uuid...
14:00:18.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.444691 29463 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.60:35027: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:18.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.499894 29603 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } attempt: 2
14:00:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.503161 29463 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } attempt: 1
14:00:18.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.505898 29603 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.60:35027: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:18.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.510046 29463 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.60:35027: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:18.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.591619 29463 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } attempt: 2
14:00:18.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.595999 29463 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.60:35027: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:18.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.597472 29603 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } attempt: 3
14:00:18.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.601905 29603 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.60:35027: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:18.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.681496 29463 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } attempt: 3
14:00:18.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.685664 29463 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.60:35027: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:18.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.745415 29603 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } attempt: 4
14:00:18.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.750984 29603 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.60:35027: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:18.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.772749 29620 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:00:18.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.773366 29620 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:00:18.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.774005 29620 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:00:18.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.808629 29620 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:00:18.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.808871 29620 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:00:18.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.809134 29620 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:00:18.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.809332 29620 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:00:18.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.809537 29620 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:00:18.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.809726 29620 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:00:18.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.809873 29620 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36509/jwks.json
14:00:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.826146 29463 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } attempt: 4
14:00:18.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.834460 29463 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.60:35027: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.853595 29620 master_runner.cc:387] Master server non-default flags:
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/logs
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.157.20:43703
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/data
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/wal
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.157.62:34669,127.28.157.61:36377,127.28.157.60:35027
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.157.60
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.157.60:35027
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36509/jwks.json
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/data/info.pb
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.157.60
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:00:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:00:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:00:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:00:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:00:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:00:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 70c51312f1186a8ae9f46647160b7730123f7601
14:00:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:00:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 13:43:22 UTC on e7f111948823
14:00:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13307
14:00:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:00:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.855113 29620 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:00:18.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:18.856654 29620 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:00:18.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.872819 29637 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:19.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:19.050449 29603 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } attempt: 5
14:00:19.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:19.059319 29603 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.60:35027: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:19.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:19.117935 29463 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } attempt: 5
14:00:19.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:19.126592 29463 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.60:35027: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:19.606871 29603 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } attempt: 6
14:00:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:19.641639 29603 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.60:35027: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:19.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:19.679106 29463 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } attempt: 6
14:00:19.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:19.717779 29463 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.60:35027: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:20.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:20.277199 29635 debug-util.cc:398] Leaking SignalData structure 0x7b0800034fe0 after lost signal to thread 29620
14:00:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:20.391108 29635 kernel_stack_watchdog.cc:198] Thread 29620 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
14:00:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:00:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:00:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:00:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:00:20.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:18.875454 29638 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:20.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:20.400519 29620 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.525s user 0.000s sys 0.008s
14:00:20.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:20.400913 29620 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.525s user 0.000s sys 0.008s
14:00:20.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:20.402016 29641 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1524 milliseconds
14:00:20.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:20.403651 29642 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:20.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.403690 29620 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:00:20.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.404982 29620 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:00:20.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.409555 29620 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:00:20.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.411022 29620 hybrid_clock.cc:648] HybridClock initialized: now 1786543220410971 us; error 67 us; skew 500 ppm
14:00:20.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.411967 29620 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:00:20.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:20 dist-test-slave-vpvm krb5kdc[29316](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543220, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:20.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.421842 29620 init.cc:377] Logged in from keytab as kudu/127.28.157.60@KRBTEST.COM (short username kudu)
14:00:20.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.431243 29620 webserver.cc:533] Webserver started at http://127.28.157.60:36163/ using document root <none> and password file <none>
14:00:20.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.432557 29620 fs_manager.cc:362] Metadata directory not provided
14:00:20.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.432862 29620 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:00:20.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.433465 29620 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:00:20.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.440340 29620 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/data/instance:
14:00:20.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27ff31c2f960440ea7d587d161dd2a1f"
14:00:20.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:20 on dist-test-slave-vpvm"
14:00:20.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.441620 29620 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/wal/instance:
14:00:20.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27ff31c2f960440ea7d587d161dd2a1f"
14:00:20.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:20 on dist-test-slave-vpvm"
14:00:20.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.449884 29620 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
14:00:20.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.456027 29663 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:00:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.457228 29620 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
14:00:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.457645 29620 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/data,/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/wal
14:00:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27ff31c2f960440ea7d587d161dd2a1f"
14:00:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:20 on dist-test-slave-vpvm"
14:00:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.458098 29620 fs_report.cc:389] FS layout report
14:00:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:00:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/wal
14:00:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/wal
14:00:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/data/data
14:00:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:00:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:00:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:00:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:00:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:00:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:00:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:00:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:00:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:00:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:00:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:00:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.541718 29620 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:00:20.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.549034 29620 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:00:20.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.549729 29620 kserver.cc:163] Server-wide thread pool size limit: 3276
14:00:20.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.702327 29603 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } attempt: 7
14:00:20.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:20.715930 29603 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.157.60:35027: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:20.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.751385 29463 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } attempt: 7
14:00:20.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.754959 29620 rpc_server.cc:307] RPC server started. Bound to: 127.28.157.60:35027
14:00:20.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.755023 29741 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.157.60:35027 every 8 connection(s)
14:00:20.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.758234 29620 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/data/info.pb
14:00:20.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.761307 29300 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu as pid 29620
14:00:20.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.761857 29300 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/wal/instance
14:00:20.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.779431 29744 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:00:20.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:20 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543215, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.62@KRBTEST.COM for kudu/127.28.157.60@KRBTEST.COM
14:00:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:20 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543213, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.157.62@KRBTEST.COM
14:00:20.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.824802 29744 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } has no permanent_uuid. Determining permanent_uuid...
14:00:20.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.860623 29463 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c: Bootstrap starting.
14:00:20.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:20 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543220, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.60@KRBTEST.COM for kudu/127.28.157.62@KRBTEST.COM
14:00:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:20 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543213, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.157.61@KRBTEST.COM
14:00:20.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.874292 29463 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c: Neither blocks nor log segments found. Creating new log.
14:00:20.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.880126 29463 log.cc:826] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c: Log is configured to *not* fsync() on all Append() calls
14:00:20.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.885318 29744 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } has no permanent_uuid. Determining permanent_uuid...
14:00:20.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.890743 29463 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c: No bootstrap required, opened a new log
14:00:20.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:20 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543213, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.157.60@KRBTEST.COM
14:00:20.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:20 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543220, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.60@KRBTEST.COM for kudu/127.28.157.61@KRBTEST.COM
14:00:20.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.922263 29744 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } has no permanent_uuid. Determining permanent_uuid...
14:00:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.927112 29463 raft_consensus.cc:359] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.928371 29463 raft_consensus.cc:385] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:00:20.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.928944 29463 raft_consensus.cc:740] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 881c1b6802e24f6d9f3fb8ca5020114c, State: Initialized, Role: FOLLOWER
14:00:20.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.931160 29463 consensus_queue.cc:260] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:20.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:20 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543220, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.60@KRBTEST.COM for kudu/127.28.157.60@KRBTEST.COM
14:00:20.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.945792 29463 sys_catalog.cc:565] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [sys.catalog]: configured and running, proceeding with master startup.
14:00:20.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.946585 29757 sys_catalog.cc:455] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:20.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.948622 29757 sys_catalog.cc:458] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [sys.catalog]: This master's current role is: FOLLOWER
14:00:20.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.968768 29744 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f: Bootstrap starting.
14:00:20.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.983042 29744 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f: Neither blocks nor log segments found. Creating new log.
14:00:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.988272 29744 log.cc:826] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f: Log is configured to *not* fsync() on all Append() calls
14:00:20.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:20.999398 29744 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f: No bootstrap required, opened a new log
14:00:21.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.040099 29744 raft_consensus.cc:359] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:21.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.041143 29744 raft_consensus.cc:385] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:00:21.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.041563 29744 raft_consensus.cc:740] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 27ff31c2f960440ea7d587d161dd2a1f, State: Initialized, Role: FOLLOWER
14:00:21.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.042695 29744 consensus_queue.cc:260] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:21.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:21.059501 29389 tablet.cc:2406] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:00:21.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.061745 29787 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:21.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.062680 29787 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [sys.catalog]: This master's current role is: FOLLOWER
14:00:21.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.063884 29744 sys_catalog.cc:565] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [sys.catalog]: configured and running, proceeding with master startup.
14:00:21.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:21.120435 29788 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:21.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:21.122219 29788 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:21.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.090027 29349 master_runner.cc:187] Error getting master registration for 127.28.157.61:36377: OK, instance_id { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" instance_seqno: 1786543218206420 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:00:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.141855 29349 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:00:21.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:21.186530 29809 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.186656 29757 raft_consensus.cc:493] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:00:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:21.186957 29809 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.187229 29757 raft_consensus.cc:515] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.210327 29704 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "27ff31c2f960440ea7d587d161dd2a1f" is_pre_election: true
14:00:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.197010 29757 leader_election.cc:290] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bab45e7c8bcd49f69477f23f98db2abf (127.28.157.61:36377), 27ff31c2f960440ea7d587d161dd2a1f (127.28.157.60:35027)
14:00:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.211414 29704 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 881c1b6802e24f6d9f3fb8ca5020114c in term 0.
14:00:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.214497 29386 leader_election.cc:304] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 27ff31c2f960440ea7d587d161dd2a1f, 881c1b6802e24f6d9f3fb8ca5020114c; no voters:
14:00:21.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.216676 29757 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [term 0 FOLLOWER]: Leader pre-election won for term 1
14:00:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.217650 29757 raft_consensus.cc:493] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:00:21.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.226703 29757 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [term 0 FOLLOWER]: Advancing to term 1
14:00:21.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.240993 29757 raft_consensus.cc:515] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:21.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.258013 29704 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "27ff31c2f960440ea7d587d161dd2a1f"
14:00:21.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.258675 29704 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [term 0 FOLLOWER]: Advancing to term 1
14:00:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.269773 29757 leader_election.cc:290] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [CANDIDATE]: Term 1 election: Requested vote from peers bab45e7c8bcd49f69477f23f98db2abf (127.28.157.61:36377), 27ff31c2f960440ea7d587d161dd2a1f (127.28.157.60:35027)
14:00:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.271325 29704 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 881c1b6802e24f6d9f3fb8ca5020114c in term 1.
14:00:21.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.273701 29386 leader_election.cc:304] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 27ff31c2f960440ea7d587d161dd2a1f, 881c1b6802e24f6d9f3fb8ca5020114c; no voters:
14:00:21.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.279639 29757 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [term 1 FOLLOWER]: Leader election won for term 1
14:00:21.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.182741 29620 master_runner.cc:187] Error getting master registration for 127.28.157.61:36377: OK, instance_id { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" instance_seqno: 1786543218206420 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:00:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.291038 29620 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:00:21.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.303009 29757 raft_consensus.cc:697] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [term 1 LEADER]: Becoming Leader. State: Replica: 881c1b6802e24f6d9f3fb8ca5020114c, State: Running, Role: LEADER
14:00:21.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.305066 29757 consensus_queue.cc:237] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:21.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:21.311368 29671 tablet.cc:2406] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:00:21.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.340737 29817 sys_catalog.cc:455] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 881c1b6802e24f6d9f3fb8ca5020114c. Latest consensus state: current_term: 1 leader_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:21.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.344067 29817 sys_catalog.cc:458] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [sys.catalog]: This master's current role is: LEADER
14:00:21.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.366498 29820 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
14:00:21.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.388027 29820 catalog_manager.cc:1520] Initializing Kudu cluster ID...
14:00:21.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.423152 29704 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [term 1 FOLLOWER]: Refusing update from remote peer 881c1b6802e24f6d9f3fb8ca5020114c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:00:21.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.425791 29817 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [LEADER]: Connected to new peer: Peer: permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:00:21.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.471203 29787 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 881c1b6802e24f6d9f3fb8ca5020114c. Latest consensus state: current_term: 1 leader_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:21.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.472260 29787 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [sys.catalog]: This master's current role is: FOLLOWER
14:00:21.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.488617 29757 sys_catalog.cc:455] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:21.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.489729 29757 sys_catalog.cc:458] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [sys.catalog]: This master's current role is: LEADER
14:00:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.495616 29787 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.496654 29787 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [sys.catalog]: This master's current role is: FOLLOWER
14:00:21.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.495116 29821 mvcc.cc:204] Tried to move back new op lower bound from 7317681034920509440 to 7317681034483511296. Current Snapshot: MvccSnapshot[applied={T|T < 7317681034920509440 or (T in {7317681034920509440})}]
14:00:21.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.510092 29820 catalog_manager.cc:1383] Generated new cluster ID: 523c54f27a854548acd6c0ca2df51452
14:00:21.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.511734 29820 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
14:00:21.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.555385 29820 catalog_manager.cc:1406] Generated new certificate authority record
14:00:21.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.563709 29820 catalog_manager.cc:1540] Loading token signing keys...
14:00:21.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.620164 29820 catalog_manager.cc:6092] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c: Generated new TSK 0
14:00:21.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.625204 29820 catalog_manager.cc:1550] Initializing in-progress tserver states...
14:00:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.157.1@KRBTEST.COM; defaulting to no policy
14:00:21.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:21.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.157.1@KRBTEST.COM" created.
14:00:21.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:21.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.157.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:21.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.157.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:21.786720 29300 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.28.157.1: real 0.133s user 0.000s sys 0.045s
14:00:21.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.157.1@KRBTEST.COM; defaulting to no policy
14:00:21.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:21.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.157.1@KRBTEST.COM" created.
14:00:21.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:21.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.157.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:21.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.157.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:21.942531 29300 mini_kdc.cc:268] Time spent creating service keytab for HTTP/127.28.157.1: real 0.155s user 0.003s sys 0.036s
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:21.945048 29300 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-0/wal
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-0/data
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-0/logs
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-0/data/info.pb
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:00:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.157.1:0
14:00:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.157.1
14:00:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.157.1
14:00:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.157.62:34669,127.28.157.61:36377,127.28.157.60:35027
14:00:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab
14:00:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.157.1
14:00:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:00:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:00:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:00:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:00:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.157.20:43703
14:00:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:00:22.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:22.199108 29809 catalog_manager.cc:1295] Loaded cluster ID: 523c54f27a854548acd6c0ca2df51452
14:00:22.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:22.199944 29809 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f: loading cluster ID for follower catalog manager: success
14:00:22.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:22.314642 29809 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f: acquiring CA information for follower catalog manager: success
14:00:22.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:22.329871 29809 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:00:22.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:22.806366 29603 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } attempt: 8
14:00:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:22.834339 29382 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.28.157.61:36377 timed out after 1.641s (SENT) [suppressed 13 similar messages]
14:00:22.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:22.864298 29835 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:00:22.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:22.865105 29835 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:00:22.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:22.866093 29835 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:00:22.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:22 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543217, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.61@KRBTEST.COM for kudu/127.28.157.60@KRBTEST.COM
14:00:22.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:22.926962 29835 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:00:22.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:22.928452 29835 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.157.1
14:00:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:22.997443 29835 tablet_server_runner.cc:78] Tablet server non-default flags:
14:00:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-0/logs
14:00:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:00:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.157.20:43703
14:00:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-0/data
14:00:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-0/wal
14:00:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab
14:00:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.157.1
14:00:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.157.1:0
14:00:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:00:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:00:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-0/data/info.pb
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.157.1
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.157.62:34669,127.28.157.61:36377,127.28.157.60:35027
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.157.1
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 70c51312f1186a8ae9f46647160b7730123f7601
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 13:43:22 UTC on e7f111948823
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13307
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:00:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:22.999236 29835 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:00:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.000762 29835 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:00:23.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:23.043601 29851 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:23.090889 29852 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:23.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.106015 29603 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf: Bootstrap starting.
14:00:23.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.132642 29603 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf: Neither blocks nor log segments found. Creating new log.
14:00:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.149415 29603 log.cc:826] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf: Log is configured to *not* fsync() on all Append() calls
14:00:23.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.180728 29603 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf: No bootstrap required, opened a new log
14:00:23.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.636631 29603 raft_consensus.cc:359] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:23.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.641198 29603 raft_consensus.cc:385] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:00:23.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.641752 29603 raft_consensus.cc:740] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bab45e7c8bcd49f69477f23f98db2abf, State: Initialized, Role: FOLLOWER
14:00:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.643402 29603 consensus_queue.cc:260] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:23.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.656001 29866 sys_catalog.cc:455] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:23.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.657351 29866 sys_catalog.cc:458] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [sys.catalog]: This master's current role is: FOLLOWER
14:00:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.662909 29603 sys_catalog.cc:565] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [sys.catalog]: configured and running, proceeding with master startup.
14:00:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.663072 29557 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bab45e7c8bcd49f69477f23f98db2abf" is_pre_election: true
14:00:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.664227 29557 raft_consensus.cc:2468] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 881c1b6802e24f6d9f3fb8ca5020114c in term 0.
14:00:23.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.667222 29558 raft_consensus.cc:3060] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 0 FOLLOWER]: Advancing to term 1
14:00:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.670506 29559 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bab45e7c8bcd49f69477f23f98db2abf"
14:00:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:23.672209 29559 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:51248 (request call id 3) took 2417 ms (client timeout 1888 ms). Trace:
14:00:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:23.672716 29559 rpcz_store.cc:269] 0812 14:00:21.255107 (+ 0us) service_pool.cc:167] Inserting onto call queue
14:00:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:21.255327 (+ 220us) service_pool.cc:224] Handling call
14:00:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:23.672085 (+2416758us) inbound_call.cc:177] Queueing success response
14:00:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:00:23.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.676738 29560 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bab45e7c8bcd49f69477f23f98db2abf" is_pre_election: true
14:00:23.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:23.678872 29560 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:51248 (request call id 2) took 2486 ms (client timeout 1641 ms). Trace:
14:00:23.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:23.679675 29560 rpcz_store.cc:269] 0812 14:00:21.192168 (+ 0us) service_pool.cc:167] Inserting onto call queue
14:00:23.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:21.192313 (+ 145us) service_pool.cc:224] Handling call
14:00:23.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:23.678757 (+2486444us) inbound_call.cc:177] Queueing success response
14:00:23.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:00:23.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.676087 29556 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bab45e7c8bcd49f69477f23f98db2abf"
14:00:23.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.708545 29558 raft_consensus.cc:1275] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 1 FOLLOWER]: Refusing update from remote peer 881c1b6802e24f6d9f3fb8ca5020114c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:00:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.709544 29558 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:51248 (request call id 4) took 2287 ms. Trace:
14:00:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.710011 29558 rpcz_store.cc:276] 0812 14:00:21.422252 (+ 0us) service_pool.cc:167] Inserting onto call queue
14:00:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:21.422408 (+ 156us) service_pool.cc:224] Handling call
14:00:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:23.666279 (+2243871us) raft_consensus.cc:1471] Updating replica for 0 ops
14:00:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:23.709145 (+ 42866us) raft_consensus.cc:1697] Filling consensus response to leader.
14:00:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:23.709464 (+ 319us) inbound_call.cc:177] Queueing success response
14:00:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:00:23.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.712828 29863 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [LEADER]: Connected to new peer: Peer: permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
14:00:23.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.874486 29866 sys_catalog.cc:455] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [sys.catalog]: SysCatalogTable state changed. Reason: New leader 881c1b6802e24f6d9f3fb8ca5020114c. Latest consensus state: current_term: 1 leader_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:23.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.875470 29866 sys_catalog.cc:458] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [sys.catalog]: This master's current role is: FOLLOWER
14:00:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.877311 29884 sys_catalog.cc:455] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:23.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.878723 29884 sys_catalog.cc:458] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [sys.catalog]: This master's current role is: FOLLOWER
14:00:23.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.946810 29889 catalog_manager.cc:1295] Loaded cluster ID: 523c54f27a854548acd6c0ca2df51452
14:00:23.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.947152 29889 catalog_manager.cc:1588] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf: loading cluster ID for follower catalog manager: success
14:00:23.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.955281 29889 catalog_manager.cc:1610] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf: acquiring CA information for follower catalog manager: success
14:00:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.961283 29889 catalog_manager.cc:1638] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:00:24.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:24.288908 29863 sys_catalog.cc:455] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:24.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:24.290506 29863 sys_catalog.cc:458] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [sys.catalog]: This master's current role is: LEADER
14:00:25.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:23.136318 29835 server_base.cc:1061] running on GCE node
14:00:25.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:23.136904 29858 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:25.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.147723 29835 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:00:25.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.150712 29835 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:00:25.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.152285 29835 hybrid_clock.cc:648] HybridClock initialized: now 1786543225152252 us; error 43 us; skew 500 ppm
14:00:25.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.153136 29835 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:00:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:25 dist-test-slave-vpvm krb5kdc[29316](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543225, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:25.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.159654 29835 init.cc:377] Logged in from keytab as kudu/127.28.157.1@KRBTEST.COM (short username kudu)
14:00:25.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.167932 29835 webserver.cc:533] Webserver started at http://127.28.157.1:43607/ using document root <none> and password file <none>
14:00:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.169303 29835 fs_manager.cc:362] Metadata directory not provided
14:00:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.169618 29835 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:00:25.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.170383 29835 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:00:25.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.177551 29835 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-0/data/instance:
14:00:25.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e3c457907be5422dacf0958be7c08ff7"
14:00:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:25 on dist-test-slave-vpvm"
14:00:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.179833 29835 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-0/wal/instance:
14:00:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e3c457907be5422dacf0958be7c08ff7"
14:00:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:25 on dist-test-slave-vpvm"
14:00:25.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.190894 29835 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.009s sys 0.001s
14:00:25.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.199419 29908 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:00:25.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.200902 29835 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.001s sys 0.004s
14:00:25.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.201416 29835 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-0/data,/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-0/wal
14:00:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e3c457907be5422dacf0958be7c08ff7"
14:00:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:25 on dist-test-slave-vpvm"
14:00:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.201840 29835 fs_report.cc:389] FS layout report
14:00:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:00:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-0/wal
14:00:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-0/wal
14:00:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-0/data/data
14:00:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:00:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:00:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:00:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:00:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:00:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:00:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:00:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:00:25.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:00:25.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:00:25.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:00:25.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.374646 29835 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:00:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.383870 29835 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:00:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.384569 29835 kserver.cc:163] Server-wide thread pool size limit: 3276
14:00:25.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.389554 29835 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:00:25.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.397120 29835 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:00:25.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.397469 29835 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:00:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.397775 29835 ts_tablet_manager.cc:616] Registered 0 tablets
14:00:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.400382 29835 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s user 0.001s sys 0.001s
14:00:25.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:25 dist-test-slave-vpvm krb5kdc[29316](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543225, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:25.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.673040 29374 init.cc:295] Successfully reacquired a new kerberos TGT
14:00:25.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.854614 29835 rpc_server.cc:307] RPC server started. Bound to: 127.28.157.1:36855
14:00:25.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.854900 30087 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.157.1:36855 every 8 connection(s)
14:00:25.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.857504 29835 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-0/data/info.pb
14:00:25.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.864248 29300 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu as pid 29835
14:00:25.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:25.864851 29300 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-0/wal/instance
14:00:25.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.157.2@KRBTEST.COM; defaulting to no policy
14:00:25.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:25.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.157.2@KRBTEST.COM" created.
14:00:25.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:25 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543225, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.1@KRBTEST.COM for kudu/127.28.157.60@KRBTEST.COM
14:00:25.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:25 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543225, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.1@KRBTEST.COM for kudu/127.28.157.61@KRBTEST.COM
14:00:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:26 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543225, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.1@KRBTEST.COM for kudu/127.28.157.62@KRBTEST.COM
14:00:26.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:26.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.157.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:26.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.157.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:26.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:26.030479 30090 heartbeater.cc:344] Connected to a master server at 127.28.157.61:36377
14:00:26.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:26.031031 30090 heartbeater.cc:461] Registering TS with master...
14:00:26.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:26.032519 30090 heartbeater.cc:507] Master 127.28.157.61:36377 requested a full tablet report, sending...
14:00:26.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:26.038028 29547 ts_manager.cc:194] Registered new tserver with Master: e3c457907be5422dacf0958be7c08ff7 (127.28.157.1:36855)
14:00:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:26.047017 30089 heartbeater.cc:344] Connected to a master server at 127.28.157.60:35027
14:00:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:26.047443 30089 heartbeater.cc:461] Registering TS with master...
14:00:26.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:26.048866 30089 heartbeater.cc:507] Master 127.28.157.60:35027 requested a full tablet report, sending...
14:00:26.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:26.053499 29691 ts_manager.cc:194] Registered new tserver with Master: e3c457907be5422dacf0958be7c08ff7 (127.28.157.1:36855)
14:00:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:26.078995 30094 heartbeater.cc:344] Connected to a master server at 127.28.157.62:34669
14:00:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:26.079548 30094 heartbeater.cc:461] Registering TS with master...
14:00:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:26.081164 30094 heartbeater.cc:507] Master 127.28.157.62:34669 requested a full tablet report, sending...
14:00:26.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:26.084789 29404 ts_manager.cc:194] Registered new tserver with Master: e3c457907be5422dacf0958be7c08ff7 (127.28.157.1:36855)
14:00:26.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:26.088790 29404 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.28.157.1@KRBTEST.COM'} at 127.28.157.1:56301
14:00:26.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.157.2@KRBTEST.COM; defaulting to no policy
14:00:26.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:26.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.157.2@KRBTEST.COM" created.
14:00:26.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:26.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.157.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:26.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.157.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:26.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:26.124557 29300 mini_kdc.cc:268] Time spent creating service keytab for HTTP/127.28.157.2: real 0.100s user 0.000s sys 0.015s
14:00:26.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:26.126376 29300 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:26.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:26.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-1/wal
14:00:26.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-1/data
14:00:26.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-1/logs
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-1/data/info.pb
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.157.2:0
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.157.2
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.157.2
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.157.62:34669,127.28.157.61:36377,127.28.157.60:35027
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.157.2
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.157.20:43703
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:00:26.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:26.728173 30111 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:00:26.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:26.728772 30111 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:00:26.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:26.729820 30111 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:00:26.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:26.786239 30111 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:00:26.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:26.787832 30111 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.157.2
14:00:26.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:26.848371 30111 tablet_server_runner.cc:78] Tablet server non-default flags:
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-1/logs
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.157.20:43703
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-1/data
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-1/wal
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.157.2
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.157.2:0
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-1/data/info.pb
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.157.2
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.157.62:34669,127.28.157.61:36377,127.28.157.60:35027
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.157.2
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:00:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:00:26.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:00:26.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 70c51312f1186a8ae9f46647160b7730123f7601
14:00:26.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:00:26.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 13:43:22 UTC on e7f111948823
14:00:26.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13307
14:00:26.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:00:26.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:26.850335 30111 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:00:26.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:26.851872 30111 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:00:26.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:26.871073 30123 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:27.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:27.094913 30094 heartbeater.cc:499] Master 127.28.157.62:34669 was elected leader, sending a full tablet report...
14:00:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:27.348161 28285 debug-util.cc:398] Leaking SignalData structure 0x7b0800094f40 after lost signal to thread 28197
14:00:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:27.350670 28285 debug-util.cc:398] Leaking SignalData structure 0x7b080008f140 after lost signal to thread 28291
14:00:27.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:27 dist-test-slave-vpvm krb5kdc[29316](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543227, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:27.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:27.975237 29514 init.cc:295] Successfully reacquired a new kerberos TGT
14:00:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:26.871886 30126 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:28.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:28.274015 30122 debug-util.cc:398] Leaking SignalData structure 0x7b0800028380 after lost signal to thread 30111
14:00:28.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:28.805651 30111 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.933s user 0.591s sys 1.045s
14:00:28.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:28.809559 30111 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.937s user 0.591s sys 1.045s
14:00:28.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:28.811563 30127 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1934 milliseconds
14:00:28.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:28.812007 30129 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:28.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.812471 30111 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:00:28.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.815347 30111 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:00:28.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.818081 30111 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:00:28.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.819532 30111 hybrid_clock.cc:648] HybridClock initialized: now 1786543228819465 us; error 85 us; skew 500 ppm
14:00:28.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.820398 30111 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:00:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:28 dist-test-slave-vpvm krb5kdc[29316](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543228, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:28.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.829288 30111 init.cc:377] Logged in from keytab as kudu/127.28.157.2@KRBTEST.COM (short username kudu)
14:00:28.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.835783 30111 webserver.cc:533] Webserver started at http://127.28.157.2:45745/ using document root <none> and password file <none>
14:00:28.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.836807 30111 fs_manager.cc:362] Metadata directory not provided
14:00:28.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.837059 30111 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:00:28.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.837668 30111 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:00:28.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.843281 30111 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-1/data/instance:
14:00:28.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2f43dc91d8564a628e22fe0c43390f9b"
14:00:28.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:28 on dist-test-slave-vpvm"
14:00:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.844724 30111 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-1/wal/instance:
14:00:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2f43dc91d8564a628e22fe0c43390f9b"
14:00:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:28 on dist-test-slave-vpvm"
14:00:28.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.856792 30111 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.010s sys 0.003s
14:00:28.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.864709 30159 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:00:28.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.866137 30111 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
14:00:28.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.866535 30111 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-1/data,/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-1/wal
14:00:28.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2f43dc91d8564a628e22fe0c43390f9b"
14:00:28.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:28 on dist-test-slave-vpvm"
14:00:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.866863 30111 fs_report.cc:389] FS layout report
14:00:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:00:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-1/wal
14:00:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-1/wal
14:00:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-1/data/data
14:00:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:00:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:00:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:00:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:00:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:00:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:00:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:00:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:00:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:00:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:00:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:00:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.921341 30111 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:00:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.927459 30111 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:00:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.927910 30111 kserver.cc:163] Server-wide thread pool size limit: 3276
14:00:28.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.931123 30111 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:00:28.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.935667 30111 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:00:28.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.935879 30111 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:00:28.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.936069 30111 ts_tablet_manager.cc:616] Registered 0 tablets
14:00:28.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:28.936203 30111 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:00:29.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.445124 30111 rpc_server.cc:307] RPC server started. Bound to: 127.28.157.2:36299
14:00:29.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.445526 30339 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.157.2:36299 every 8 connection(s)
14:00:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.449738 30111 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-1/data/info.pb
14:00:29.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.456107 29300 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu as pid 30111
14:00:29.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.456664 29300 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-1/wal/instance
14:00:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.157.3@KRBTEST.COM; defaulting to no policy
14:00:29.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:29.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.157.3@KRBTEST.COM" created.
14:00:29.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:29.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.157.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:29.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.157.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:29.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:29 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543228, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.2@KRBTEST.COM for kudu/127.28.157.61@KRBTEST.COM
14:00:29.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:29 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543228, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.2@KRBTEST.COM for kudu/127.28.157.60@KRBTEST.COM
14:00:29.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:29 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543228, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.2@KRBTEST.COM for kudu/127.28.157.62@KRBTEST.COM
14:00:29.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.606027 30341 heartbeater.cc:344] Connected to a master server at 127.28.157.61:36377
14:00:29.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.606746 30341 heartbeater.cc:461] Registering TS with master...
14:00:29.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.609382 30341 heartbeater.cc:507] Master 127.28.157.61:36377 requested a full tablet report, sending...
14:00:29.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.157.3@KRBTEST.COM; defaulting to no policy
14:00:29.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.615288 29547 ts_manager.cc:194] Registered new tserver with Master: 2f43dc91d8564a628e22fe0c43390f9b (127.28.157.2:36299)
14:00:29.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:29.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.157.3@KRBTEST.COM" created.
14:00:29.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.627676 30352 heartbeater.cc:344] Connected to a master server at 127.28.157.62:34669
14:00:29.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.628032 30352 heartbeater.cc:461] Registering TS with master...
14:00:29.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.629032 30352 heartbeater.cc:507] Master 127.28.157.62:34669 requested a full tablet report, sending...
14:00:29.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.632079 29404 ts_manager.cc:194] Registered new tserver with Master: 2f43dc91d8564a628e22fe0c43390f9b (127.28.157.2:36299)
14:00:29.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.634400 29404 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.28.157.2@KRBTEST.COM'} at 127.28.157.2:53377
14:00:29.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.638528 30340 heartbeater.cc:344] Connected to a master server at 127.28.157.60:35027
14:00:29.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.639072 30340 heartbeater.cc:461] Registering TS with master...
14:00:29.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.640205 30340 heartbeater.cc:507] Master 127.28.157.60:35027 requested a full tablet report, sending...
14:00:29.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.644888 29691 ts_manager.cc:194] Registered new tserver with Master: 2f43dc91d8564a628e22fe0c43390f9b (127.28.157.2:36299)
14:00:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:00:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.157.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.157.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab.
14:00:29.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:29.660522 29300 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:29.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:29.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-2/wal
14:00:29.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-2/data
14:00:29.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:00:29.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:00:29.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:00:29.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:29.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:29.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:29.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-2/logs
14:00:29.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-2/data/info.pb
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.157.3:0
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.157.3
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.157.3
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.157.62:34669,127.28.157.61:36377,127.28.157.60:35027
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.157.3
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.157.20:43703
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:00:30.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:30.257236 30365 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:00:30.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:30.258008 30365 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:00:30.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:30.258706 30365 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:00:30.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:30.294373 30365 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:00:30.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:30.295499 30365 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.157.3
14:00:30.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:30.336361 30365 tablet_server_runner.cc:78] Tablet server non-default flags:
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-2/logs
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.157.20:43703
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-2/data
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-2/wal
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.157.3
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.157.3:0
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-2/data/info.pb
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.157.3
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.157.62:34669,127.28.157.61:36377,127.28.157.60:35027
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.157.3
14:00:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:00:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:00:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:00:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 70c51312f1186a8ae9f46647160b7730123f7601
14:00:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:00:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 13:43:22 UTC on e7f111948823
14:00:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13307
14:00:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:00:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:30.338158 30365 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:00:30.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:30.339458 30365 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:00:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:30.361630 30373 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:30.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:30 dist-test-slave-vpvm krb5kdc[29316](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543230, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:30.432855 29654 init.cc:295] Successfully reacquired a new kerberos TGT
14:00:30.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:30.641534 30352 heartbeater.cc:499] Master 127.28.157.62:34669 was elected leader, sending a full tablet report...
14:00:31.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:31.765717 30372 debug-util.cc:398] Leaking SignalData structure 0x7b0800028380 after lost signal to thread 30365
14:00:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:31.821022 30372 kernel_stack_watchdog.cc:198] Thread 30365 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 390ms:
14:00:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:00:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:00:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:00:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:00:31.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:30.366405 30374 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:31.831210 30365 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.461s user 0.004s sys 0.007s
14:00:31.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:31.831653 30365 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.461s user 0.004s sys 0.007s
14:00:31.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:31.834180 30378 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:31.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:31.835222 30365 server_base.cc:1061] running on GCE node
14:00:31.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:31.836723 30365 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:00:31.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:31.840283 30365 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:00:31.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:31.841632 30365 hybrid_clock.cc:648] HybridClock initialized: now 1786543231841592 us; error 48 us; skew 500 ppm
14:00:31.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:31.842595 30365 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:00:31.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:31 dist-test-slave-vpvm krb5kdc[29316](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543231, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:31.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:31.856411 30365 init.cc:377] Logged in from keytab as kudu/127.28.157.3@KRBTEST.COM (short username kudu)
14:00:31.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:31.870729 30365 webserver.cc:533] Webserver started at http://127.28.157.3:39873/ using document root <none> and password file <none>
14:00:31.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:31.871810 30365 fs_manager.cc:362] Metadata directory not provided
14:00:31.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:31.872088 30365 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:00:31.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:31.872686 30365 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:00:31.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:31.880110 30365 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-2/data/instance:
14:00:31.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2848775df46846bb80bcba347c10f984"
14:00:31.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:31 on dist-test-slave-vpvm"
14:00:31.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:31.881800 30365 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-2/wal/instance:
14:00:31.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2848775df46846bb80bcba347c10f984"
14:00:31.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:31 on dist-test-slave-vpvm"
14:00:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:31.898023 30365 fs_manager.cc:696] Time spent creating directory manager: real 0.015s user 0.009s sys 0.005s
14:00:31.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:31.911093 30389 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:00:31.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:31.912719 30365 fs_manager.cc:730] Time spent opening block manager: real 0.009s user 0.004s sys 0.004s
14:00:31.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:31.913102 30365 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-2/data,/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-2/wal
14:00:31.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2848775df46846bb80bcba347c10f984"
14:00:31.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:31 on dist-test-slave-vpvm"
14:00:31.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:31.913550 30365 fs_report.cc:389] FS layout report
14:00:31.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:00:31.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-2/wal
14:00:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-2/wal
14:00:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-2/data/data
14:00:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:00:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:00:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:00:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:00:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:00:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:00:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:00:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:00:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:00:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:00:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:00:31.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:31.993767 30365 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:00:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.007733 30365 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:00:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.008392 30365 kserver.cc:163] Server-wide thread pool size limit: 3276
14:00:32.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.016799 30365 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:00:32.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.026063 30365 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:00:32.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.026401 30365 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:00:32.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.026744 30365 ts_tablet_manager.cc:616] Registered 0 tablets
14:00:32.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.026906 30365 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:00:32.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.620648 30365 rpc_server.cc:307] RPC server started. Bound to: 127.28.157.3:34613
14:00:32.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.620815 30566 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.157.3:34613 every 8 connection(s)
14:00:32.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.624119 30365 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-2/data/info.pb
14:00:32.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.627655 29300 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu as pid 30365
14:00:32.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.628757 29300 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/ts-2/wal/instance
14:00:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:32 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543231, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.3@KRBTEST.COM for kudu/127.28.157.62@KRBTEST.COM
14:00:32.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:32 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543231, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.3@KRBTEST.COM for kudu/127.28.157.61@KRBTEST.COM
14:00:32.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:32 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543231, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.3@KRBTEST.COM for kudu/127.28.157.60@KRBTEST.COM
14:00:32.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.753597 30568 heartbeater.cc:344] Connected to a master server at 127.28.157.61:36377
14:00:32.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.753916 30573 heartbeater.cc:344] Connected to a master server at 127.28.157.62:34669
14:00:32.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.754343 30568 heartbeater.cc:461] Registering TS with master...
14:00:32.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.754570 30573 heartbeater.cc:461] Registering TS with master...
14:00:32.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.757071 30568 heartbeater.cc:507] Master 127.28.157.61:36377 requested a full tablet report, sending...
14:00:32.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.759302 30573 heartbeater.cc:507] Master 127.28.157.62:34669 requested a full tablet report, sending...
14:00:32.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.759684 30567 heartbeater.cc:344] Connected to a master server at 127.28.157.60:35027
14:00:32.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.760217 30567 heartbeater.cc:461] Registering TS with master...
14:00:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.764024 30567 heartbeater.cc:507] Master 127.28.157.60:35027 requested a full tablet report, sending...
14:00:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.768941 29547 ts_manager.cc:194] Registered new tserver with Master: 2848775df46846bb80bcba347c10f984 (127.28.157.3:34613)
14:00:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.769030 29404 ts_manager.cc:194] Registered new tserver with Master: 2848775df46846bb80bcba347c10f984 (127.28.157.3:34613)
14:00:32.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.776904 29404 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.28.157.3@KRBTEST.COM'} at 127.28.157.3:48037
14:00:32.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.778373 29687 ts_manager.cc:194] Registered new tserver with Master: 2848775df46846bb80bcba347c10f984 (127.28.157.3:34613)
14:00:32.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.802415 29300 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
14:00:32.813 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:00:32.816 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
14:00:32.824 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
14:00:32.826 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
14:00:32.829 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.28.157.62"
port: 34669
}
bound_http_address {
host: "127.28.157.62"
port: 36765
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.28.157.61"
port: 36377
}
bound_http_address {
host: "127.28.157.61"
port: 39947
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.28.157.60"
port: 35027
}
bound_http_address {
host: "127.28.157.60"
port: 36163
}
}
}
14:00:32.829 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:00:32.831 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.28.157.1"
port: 36855
}
bound_http_address {
host: "127.28.157.1"
port: 43607
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.28.157.2"
port: 36299
}
bound_http_address {
host: "127.28.157.2"
port: 45745
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.28.157.3"
port: 34613
}
bound_http_address {
host: "127.28.157.3"
port: 39873
}
}
}
14:00:32.831 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:00:32.832 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:00:32.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:32 dist-test-slave-vpvm krb5kdc[29316](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543232, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:32.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:00:32.881 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:00:32.884 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5cc
14:00:32.896 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:00:32.896 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5cc
14:00:32.898 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:00:32.927 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
14:00:32.928 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:00:32.929 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:00:32.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:32 dist-test-slave-vpvm krb5kdc[29316](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543232, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:32.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:00:32.968 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:00:32.968 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5cc
14:00:32.970 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:00:32.970 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.157.61:36377
14:00:32.972 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:00:32.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:32.973078 29300 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu with pid 29475
14:00:33.023 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:00:33.024 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.157.62:34669
14:00:33.024 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:00:33.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:33.024606 29300 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu with pid 29349
14:00:33.068 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:00:33.069 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.157.60:35027
14:00:33.069 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:00:33.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:33.069875 29300 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu with pid 29620
14:00:33.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:33.086057 30090 heartbeater.cc:646] Failed to heartbeat to 127.28.157.61:36377 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.28.157.61:36377: connect: Connection refused (error 111)
14:00:33.103 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:00:33.104 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.157.61:36377
14:00:33.107 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:00:33.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:33.109438 29300 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:33.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:33.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/wal
14:00:33.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/data
14:00:33.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:00:33.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:00:33.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:00:33.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:33.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:33.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:33.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/logs
14:00:33.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/data/info.pb
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.157.61:36377
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.157.61
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39947
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.157.61
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.157.20:43703
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:00:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.157.62:34669,127.28.157.61:36377,127.28.157.60:35027
14:00:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:00:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36509/jwks.json
14:00:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:00:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:00:33.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:33.473816 30591 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:00:33.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:33.474681 30591 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:00:33.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:33.475742 30591 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:00:33.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:33.514695 30591 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:00:33.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:33.514986 30591 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:00:33.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:33.515393 30591 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:00:33.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:33.515707 30591 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:00:33.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:33.516019 30591 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:00:33.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:33.516321 30591 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:00:33.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:33.516551 30591 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36509/jwks.json
14:00:33.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:33.557242 30591 master_runner.cc:387] Master server non-default flags:
14:00:33.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/logs
14:00:33.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.157.20:43703
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/data
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/wal
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.157.62:34669,127.28.157.61:36377,127.28.157.60:35027
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.157.61
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.157.61:36377
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36509/jwks.json
14:00:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/data/info.pb
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.157.61
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39947
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 70c51312f1186a8ae9f46647160b7730123f7601
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 13:43:22 UTC on e7f111948823
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13307
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:33.559079 30591 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:00:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:33.560617 30591 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:00:33.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:33.572911 30603 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:33.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:33.654398 30341 heartbeater.cc:646] Failed to heartbeat to 127.28.157.61:36377 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.28.157.61:36377: connect: Connection refused (error 111)
14:00:33.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:33.786135 30573 heartbeater.cc:499] Master 127.28.157.62:34669 was elected leader, sending a full tablet report...
14:00:33.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:33.798735 30567 heartbeater.cc:646] Failed to heartbeat to 127.28.157.60:35027 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:34.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:34.435307 30081 debug-util.cc:398] Leaking SignalData structure 0x7b08000adca0 after lost signal to thread 29839
14:00:34.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:34.436436 30081 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba720 after lost signal to thread 30087
14:00:35.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:35 dist-test-slave-vpvm krb5kdc[29316](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543235, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:35.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.165836 29899 init.cc:295] Successfully reacquired a new kerberos TGT
14:00:35.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:34.977012 30602 debug-util.cc:398] Leaking SignalData structure 0x7b0800034fe0 after lost signal to thread 30591
14:00:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:35.194438 30602 kernel_stack_watchdog.cc:198] Thread 30591 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
14:00:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:00:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:00:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:00:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:00:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:33.574223 30605 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:35.196511 30608 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:35.198140 30591 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.624s user 0.561s sys 0.873s
14:00:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:35.199602 30591 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.626s user 0.561s sys 0.873s
14:00:35.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:35.204283 30607 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1624 milliseconds
14:00:35.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.204326 30591 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:00:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.206557 30591 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:00:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.210005 30591 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:00:35.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.211548 30591 hybrid_clock.cc:648] HybridClock initialized: now 1786543235211403 us; error 157 us; skew 500 ppm
14:00:35.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.212457 30591 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:00:35.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:35 dist-test-slave-vpvm krb5kdc[29316](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543235, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.219095 30591 init.cc:377] Logged in from keytab as kudu/127.28.157.61@KRBTEST.COM (short username kudu)
14:00:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.225248 30591 webserver.cc:533] Webserver started at http://127.28.157.61:39947/ using document root <none> and password file <none>
14:00:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.226243 30591 fs_manager.cc:362] Metadata directory not provided
14:00:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.226475 30591 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:00:35.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.235040 30591 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.004s sys 0.003s
14:00:35.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.239832 30638 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:00:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.241051 30591 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
14:00:35.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.241475 30591 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/data,/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/wal
14:00:35.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bab45e7c8bcd49f69477f23f98db2abf"
14:00:35.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:17 on dist-test-slave-vpvm"
14:00:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.243641 30591 fs_report.cc:389] FS layout report
14:00:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:00:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/wal
14:00:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/wal
14:00:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/data/data
14:00:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:00:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:00:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:00:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:00:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:00:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:00:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:00:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:00:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:00:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:00:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:00:35.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.326948 30591 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:00:35.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.332998 30591 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:00:35.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.333455 30591 kserver.cc:163] Server-wide thread pool size limit: 3276
14:00:35.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.419073 30591 rpc_server.cc:307] RPC server started. Bound to: 127.28.157.61:36377
14:00:35.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.419229 30721 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.157.61:36377 every 8 connection(s)
14:00:35.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.422147 30591 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-1/data/info.pb
14:00:35.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.427187 29300 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu as pid 30591
14:00:35.427 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:00:35.428 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.157.62:34669
14:00:35.428 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:00:35.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.430193 29300 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:35.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:35.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/wal
14:00:35.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/data
14:00:35.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:00:35.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:00:35.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:00:35.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:35.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:35.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/logs
14:00:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/data/info.pb
14:00:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:00:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:00:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:00:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:00:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:00:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:00:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:00:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:00:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.157.62:34669
14:00:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.157.62
14:00:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36765
14:00:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab
14:00:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.157.62
14:00:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:00:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:00:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:00:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:00:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.157.20:43703
14:00:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:00:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.157.62:34669,127.28.157.61:36377,127.28.157.60:35027
14:00:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:00:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36509/jwks.json
14:00:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:00:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:00:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.452567 30724 sys_catalog.cc:263] Verifying existing consensus state
14:00:35.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.461429 30724 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf: Bootstrap starting.
14:00:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.517812 30724 log.cc:826] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf: Log is configured to *not* fsync() on all Append() calls
14:00:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.538359 30724 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:00:35.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.539430 30724 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf: Bootstrap complete.
14:00:35.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.574360 30724 raft_consensus.cc:359] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:35.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.577857 30724 raft_consensus.cc:740] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: bab45e7c8bcd49f69477f23f98db2abf, State: Initialized, Role: FOLLOWER
14:00:35.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.579128 30724 consensus_queue.cc:260] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:35.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.583252 30732 sys_catalog.cc:455] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:35.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.584515 30732 sys_catalog.cc:458] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [sys.catalog]: This master's current role is: FOLLOWER
14:00:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.585357 30724 sys_catalog.cc:565] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [sys.catalog]: configured and running, proceeding with master startup.
14:00:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.633579 30748 catalog_manager.cc:1295] Loaded cluster ID: 523c54f27a854548acd6c0ca2df51452
14:00:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.634402 30748 catalog_manager.cc:1588] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf: loading cluster ID for follower catalog manager: success
14:00:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.643607 30748 catalog_manager.cc:1610] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf: acquiring CA information for follower catalog manager: success
14:00:35.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.651710 30748 catalog_manager.cc:1638] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:00:35.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.628062 30591 master_runner.cc:187] Error getting master registration for 127.28.157.62:34669: Network error: Client connection negotiation failed: client connection to 127.28.157.62:34669: connect: Connection refused (error 111),
14:00:35.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.668190 30591 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:00:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.780009 30341 heartbeater.cc:344] Connected to a master server at 127.28.157.61:36377
14:00:35.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.789812 30669 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2f43dc91d8564a628e22fe0c43390f9b" instance_seqno: 1786543229290644) as {username='kudu', principal='kudu/127.28.157.2@KRBTEST.COM'} at 127.28.157.2:47065; Asking this server to re-register.
14:00:35.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.792826 30341 heartbeater.cc:461] Registering TS with master...
14:00:35.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.797641 30667 ts_manager.cc:194] Registered new tserver with Master: 2f43dc91d8564a628e22fe0c43390f9b (127.28.157.2:36299)
14:00:35.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.868429 30568 heartbeater.cc:344] Connected to a master server at 127.28.157.61:36377
14:00:35.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.871824 30669 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2848775df46846bb80bcba347c10f984" instance_seqno: 1786543232497729) as {username='kudu', principal='kudu/127.28.157.3@KRBTEST.COM'} at 127.28.157.3:39055; Asking this server to re-register.
14:00:35.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.874753 30568 heartbeater.cc:461] Registering TS with master...
14:00:35.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:35.878928 30669 ts_manager.cc:194] Registered new tserver with Master: 2848775df46846bb80bcba347c10f984 (127.28.157.3:34613)
14:00:35.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:35.981392 30729 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:00:35.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:35.982244 30729 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:00:35.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:35.983386 30729 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:00:36.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:36.043555 30729 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:00:36.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:36.043879 30729 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:00:36.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:36.044332 30729 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:00:36.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:36.044675 30729 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:00:36.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:36.045001 30729 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:00:36.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:36.045311 30729 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:00:36.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:36.045547 30729 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36509/jwks.json
14:00:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:36.112082 30729 master_runner.cc:387] Master server non-default flags:
14:00:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/logs
14:00:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:00:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.157.20:43703
14:00:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/data
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/wal
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.157.62:34669,127.28.157.61:36377,127.28.157.60:35027
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.157.62
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.157.62:34669
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36509/jwks.json
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/data/info.pb
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.157.62
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36765
14:00:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:00:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:00:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:00:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:00:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:00:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:00:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 70c51312f1186a8ae9f46647160b7730123f7601
14:00:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:00:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 13:43:22 UTC on e7f111948823
14:00:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13307
14:00:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:00:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:36.113822 30729 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:00:36.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:36.115324 30729 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:00:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:36.165637 30090 heartbeater.cc:344] Connected to a master server at 127.28.157.61:36377
14:00:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:36.167980 30782 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:36.168542 30669 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "e3c457907be5422dacf0958be7c08ff7" instance_seqno: 1786543225728517) as {username='kudu', principal='kudu/127.28.157.1@KRBTEST.COM'} at 127.28.157.1:46215; Asking this server to re-register.
14:00:36.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:36.169981 30090 heartbeater.cc:461] Registering TS with master...
14:00:36.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:36.173225 30669 ts_manager.cc:194] Registered new tserver with Master: e3c457907be5422dacf0958be7c08ff7 (127.28.157.1:36855)
14:00:36.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:36.744273 30792 raft_consensus.cc:493] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:00:36.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:36.744999 30792 raft_consensus.cc:515] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:36.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:36.768930 30792 leader_election.cc:290] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 881c1b6802e24f6d9f3fb8ca5020114c (127.28.157.62:34669), 27ff31c2f960440ea7d587d161dd2a1f (127.28.157.60:35027)
14:00:36.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:36.804581 30650 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:36.817866 30641 leader_election.cc:336] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 881c1b6802e24f6d9f3fb8ca5020114c (127.28.157.62:34669): Network error: Client connection negotiation failed: client connection to 127.28.157.62:34669: connect: Connection refused (error 111)
14:00:36.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:36.822728 30650 leader_election.cc:336] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 27ff31c2f960440ea7d587d161dd2a1f (127.28.157.60:35027): Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:36.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:36.823737 30650 leader_election.cc:304] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bab45e7c8bcd49f69477f23f98db2abf; no voters: 27ff31c2f960440ea7d587d161dd2a1f, 881c1b6802e24f6d9f3fb8ca5020114c
14:00:36.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:36.827068 30792 raft_consensus.cc:2749] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:00:37.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:37.551064 30777 debug-util.cc:398] Leaking SignalData structure 0x7b0800034fe0 after lost signal to thread 30729
14:00:37.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:37.786110 30729 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.622s user 0.638s sys 0.818s
14:00:37.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:37.787190 30729 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.623s user 0.638s sys 0.818s
14:00:37.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:36.169670 30783 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:37.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:37.801177 30805 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:37.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:37.805166 30729 server_base.cc:1061] running on GCE node
14:00:37.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:37.806712 30729 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:00:37.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:37.809464 30729 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:00:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:37.811080 30729 hybrid_clock.cc:648] HybridClock initialized: now 1786543237811008 us; error 86 us; skew 500 ppm
14:00:37.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:37.811892 30729 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:00:37.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:37 dist-test-slave-vpvm krb5kdc[29316](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543237, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:37.819070 30729 init.cc:377] Logged in from keytab as kudu/127.28.157.62@KRBTEST.COM (short username kudu)
14:00:37.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:37.824995 30729 webserver.cc:533] Webserver started at http://127.28.157.62:36765/ using document root <none> and password file <none>
14:00:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:37.826333 30729 fs_manager.cc:362] Metadata directory not provided
14:00:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:37.826655 30729 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:00:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:37.838053 30729 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.007s sys 0.000s
14:00:37.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:37.844175 30819 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:00:37.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:37.845551 30729 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
14:00:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:37.845920 30729 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/data,/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/wal
14:00:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "881c1b6802e24f6d9f3fb8ca5020114c"
14:00:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:15 on dist-test-slave-vpvm"
14:00:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:37.848242 30729 fs_report.cc:389] FS layout report
14:00:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:00:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/wal
14:00:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/wal
14:00:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/data/data
14:00:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:00:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:00:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:00:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:00:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:00:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:00:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:00:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:00:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:00:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:00:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:00:37.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:37.931017 30729 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:00:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:37.940816 30729 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:00:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:37.941381 30729 kserver.cc:163] Server-wide thread pool size limit: 3276
14:00:38.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.029488 30729 rpc_server.cc:307] RPC server started. Bound to: 127.28.157.62:34669
14:00:38.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.029577 30898 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.157.62:34669 every 8 connection(s)
14:00:38.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.032529 30729 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-0/data/info.pb
14:00:38.035 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:00:38.036 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.157.60:35027
14:00:38.036 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:00:38.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.034485 29300 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu as pid 30729
14:00:38.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.041316 29300 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:38.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu
14:00:38.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/wal
14:00:38.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/data
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/logs
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/data/info.pb
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.157.60:35027
14:00:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.157.60
14:00:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36163
14:00:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab
14:00:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.157.60
14:00:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:00:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:00:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:00:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:00:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.157.20:43703
14:00:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:00:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.157.62:34669,127.28.157.61:36377,127.28.157.60:35027
14:00:38.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:00:38.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36509/jwks.json
14:00:38.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:00:38.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:00:38.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.048173 30900 sys_catalog.cc:263] Verifying existing consensus state
14:00:38.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.056476 30900 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c: Bootstrap starting.
14:00:38.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.112088 30900 log.cc:826] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c: Log is configured to *not* fsync() on all Append() calls
14:00:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.128760 30900 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:00:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.129638 30900 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c: Bootstrap complete.
14:00:38.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.158800 30900 raft_consensus.cc:359] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:38.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.162147 30900 raft_consensus.cc:740] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 881c1b6802e24f6d9f3fb8ca5020114c, State: Initialized, Role: FOLLOWER
14:00:38.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.163348 30900 consensus_queue.cc:260] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:38.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.168308 30900 sys_catalog.cc:565] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [sys.catalog]: configured and running, proceeding with master startup.
14:00:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.166414 30905 sys_catalog.cc:455] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:38.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.180938 30905 sys_catalog.cc:458] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [sys.catalog]: This master's current role is: FOLLOWER
14:00:38.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:38 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543235, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.1@KRBTEST.COM for kudu/127.28.157.62@KRBTEST.COM
14:00:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.246381 30094 heartbeater.cc:344] Connected to a master server at 127.28.157.62:34669
14:00:38.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.284515 30930 catalog_manager.cc:1295] Loaded cluster ID: 523c54f27a854548acd6c0ca2df51452
14:00:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.284829 30930 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c: loading cluster ID for follower catalog manager: success
14:00:38.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:38 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543237, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.62@KRBTEST.COM for kudu/127.28.157.62@KRBTEST.COM
14:00:38.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.308068 30930 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c: acquiring CA information for follower catalog manager: success
14:00:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.313894 30930 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:00:38.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.391664 30729 master_runner.cc:187] Error getting master registration for 127.28.157.60:35027: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111),
14:00:38.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.420572 30729 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:00:38.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.501251 30938 raft_consensus.cc:493] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:00:38.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.502321 30938 raft_consensus.cc:515] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:38.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.533193 30938 leader_election.cc:290] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 881c1b6802e24f6d9f3fb8ca5020114c (127.28.157.62:34669), 27ff31c2f960440ea7d587d161dd2a1f (127.28.157.60:35027)
14:00:38.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:38.558180 30650 leader_election.cc:336] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 27ff31c2f960440ea7d587d161dd2a1f (127.28.157.60:35027): Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.565840 30857 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bab45e7c8bcd49f69477f23f98db2abf" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" is_pre_election: true
14:00:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.566915 30857 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bab45e7c8bcd49f69477f23f98db2abf in term 1.
14:00:38.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.568629 30641 leader_election.cc:304] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 881c1b6802e24f6d9f3fb8ca5020114c, bab45e7c8bcd49f69477f23f98db2abf; no voters: 27ff31c2f960440ea7d587d161dd2a1f
14:00:38.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.569685 30938 raft_consensus.cc:2804] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 1 FOLLOWER]: Leader pre-election won for term 2
14:00:38.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.570039 30938 raft_consensus.cc:493] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:00:38.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.570362 30938 raft_consensus.cc:3060] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 1 FOLLOWER]: Advancing to term 2
14:00:38.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.580757 30938 raft_consensus.cc:515] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:38.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.584079 30857 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bab45e7c8bcd49f69477f23f98db2abf" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "881c1b6802e24f6d9f3fb8ca5020114c"
14:00:38.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.584638 30857 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [term 1 FOLLOWER]: Advancing to term 2
14:00:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.596366 30857 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bab45e7c8bcd49f69477f23f98db2abf in term 2.
14:00:38.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:38.600126 30902 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:00:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:38.600898 30902 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:00:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:38.601965 30902 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:00:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:38.603902 30650 leader_election.cc:336] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 27ff31c2f960440ea7d587d161dd2a1f (127.28.157.60:35027): Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111)
14:00:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.601768 30938 leader_election.cc:290] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [CANDIDATE]: Term 2 election: Requested vote from peers 881c1b6802e24f6d9f3fb8ca5020114c (127.28.157.62:34669), 27ff31c2f960440ea7d587d161dd2a1f (127.28.157.60:35027)
14:00:38.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.606145 30641 leader_election.cc:304] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 881c1b6802e24f6d9f3fb8ca5020114c, bab45e7c8bcd49f69477f23f98db2abf; no voters: 27ff31c2f960440ea7d587d161dd2a1f
14:00:38.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.607417 30938 raft_consensus.cc:2804] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 2 FOLLOWER]: Leader election won for term 2
14:00:38.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.613394 30938 raft_consensus.cc:697] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [term 2 LEADER]: Becoming Leader. State: Replica: bab45e7c8bcd49f69477f23f98db2abf, State: Running, Role: LEADER
14:00:38.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.614635 30938 consensus_queue.cc:237] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:38.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.630690 30957 sys_catalog.cc:455] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [sys.catalog]: SysCatalogTable state changed. Reason: New leader bab45e7c8bcd49f69477f23f98db2abf. Latest consensus state: current_term: 2 leader_uuid: "bab45e7c8bcd49f69477f23f98db2abf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:38.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.631875 30957 sys_catalog.cc:458] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [sys.catalog]: This master's current role is: LEADER
14:00:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.641345 30961 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
14:00:38.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.652194 30961 catalog_manager.cc:1520] Initializing Kudu cluster ID...
14:00:38.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.657258 30961 catalog_manager.cc:1295] Loaded cluster ID: 523c54f27a854548acd6c0ca2df51452
14:00:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.657887 30961 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
14:00:38.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.663322 30961 catalog_manager.cc:1540] Loading token signing keys...
14:00:38.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.668007 30961 catalog_manager.cc:6103] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf: Loaded TSK: 0
14:00:38.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.669579 30961 catalog_manager.cc:1550] Initializing in-progress tserver states...
14:00:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:38.707137 30902 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:00:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:38.707494 30902 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:00:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:38.707994 30902 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:00:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:38.708364 30902 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:00:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:38.708894 30902 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:00:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:38.709260 30902 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:00:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:38.709533 30902 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36509/jwks.json
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.784444 30902 master_runner.cc:387] Master server non-default flags:
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/logs
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.157.20:43703
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/data
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/wal
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.157.62:34669,127.28.157.61:36377,127.28.157.60:35027
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/krb5kdc/kudu.keytab
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.157.60
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.157.60:35027
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:00:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36509/jwks.json
14:00:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:00:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:00:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:00:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:00:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:00:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/data/info.pb
14:00:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.157.60
14:00:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36163
14:00:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:00:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:00:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:00:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:00:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:00:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:00:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:00:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:00:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 70c51312f1186a8ae9f46647160b7730123f7601
14:00:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:00:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 13:43:22 UTC on e7f111948823
14:00:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13307
14:00:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:00:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.786213 30902 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:00:38.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.787721 30902 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:00:38.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:38.809943 30965 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:38.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.817227 30352 heartbeater.cc:344] Connected to a master server at 127.28.157.62:34669
14:00:38.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.823546 30842 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2f43dc91d8564a628e22fe0c43390f9b" instance_seqno: 1786543229290644) as {username='kudu', principal='kudu/127.28.157.2@KRBTEST.COM'} at 127.28.157.2:52237; Asking this server to re-register.
14:00:38.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.825246 30352 heartbeater.cc:461] Registering TS with master...
14:00:38.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.829022 30842 ts_manager.cc:194] Registered new tserver with Master: 2f43dc91d8564a628e22fe0c43390f9b (127.28.157.2:36299)
14:00:38.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:38 dist-test-slave-vpvm krb5kdc[29316](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543238, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:38.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.835146 30149 init.cc:295] Successfully reacquired a new kerberos TGT
14:00:38.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.943585 30573 heartbeater.cc:344] Connected to a master server at 127.28.157.62:34669
14:00:38.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.944244 30573 heartbeater.cc:499] Master 127.28.157.62:34669 was elected leader, sending a full tablet report...
14:00:38.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.947227 30842 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2848775df46846bb80bcba347c10f984" instance_seqno: 1786543232497729) as {username='kudu', principal='kudu/127.28.157.3@KRBTEST.COM'} at 127.28.157.3:47951; Asking this server to re-register.
14:00:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.949209 30573 heartbeater.cc:461] Registering TS with master...
14:00:38.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:38.953213 30842 ts_manager.cc:194] Registered new tserver with Master: 2848775df46846bb80bcba347c10f984 (127.28.157.3:34613)
14:00:39.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:39.009272 30650 consensus_peers.cc:597] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf -> Peer 27ff31c2f960440ea7d587d161dd2a1f (127.28.157.60:35027): Couldn't send request to peer 27ff31c2f960440ea7d587d161dd2a1f. Status: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:00:39.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:39.036537 30857 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [term 2 FOLLOWER]: Refusing update from remote peer bab45e7c8bcd49f69477f23f98db2abf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
14:00:39.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:39.038555 30957 consensus_queue.cc:1048] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [LEADER]: Connected to new peer: Peer: permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
14:00:39.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:39.078699 30977 sys_catalog.cc:455] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [sys.catalog]: SysCatalogTable state changed. Reason: New leader bab45e7c8bcd49f69477f23f98db2abf. Latest consensus state: current_term: 2 leader_uuid: "bab45e7c8bcd49f69477f23f98db2abf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:39.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:39.081003 30977 sys_catalog.cc:458] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [sys.catalog]: This master's current role is: FOLLOWER
14:00:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:39.097545 30957 sys_catalog.cc:455] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "bab45e7c8bcd49f69477f23f98db2abf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:39.098392 30957 sys_catalog.cc:458] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [sys.catalog]: This master's current role is: LEADER
14:00:39.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:39.100950 30977 sys_catalog.cc:455] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "bab45e7c8bcd49f69477f23f98db2abf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:39.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:39.101701 30977 sys_catalog.cc:458] T 00000000000000000000000000000000 P 881c1b6802e24f6d9f3fb8ca5020114c [sys.catalog]: This master's current role is: FOLLOWER
14:00:39.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:39.263725 30842 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "e3c457907be5422dacf0958be7c08ff7" instance_seqno: 1786543225728517) as {username='kudu', principal='kudu/127.28.157.1@KRBTEST.COM'} at 127.28.157.1:49887; Asking this server to re-register.
14:00:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:39.266396 30094 heartbeater.cc:461] Registering TS with master...
14:00:39.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:39.271035 30842 ts_manager.cc:194] Registered new tserver with Master: e3c457907be5422dacf0958be7c08ff7 (127.28.157.1:36855)
14:00:39.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:39.727125 28397 debug-util.cc:398] Leaking SignalData structure 0x7b0800099ca0 after lost signal to thread 28300
14:00:39.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:39.728442 28397 debug-util.cc:398] Leaking SignalData structure 0x7b0800070300 after lost signal to thread 28401
14:00:39.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:39.823320 30341 heartbeater.cc:499] Master 127.28.157.61:36377 was elected leader, sending a full tablet report...
14:00:39.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:39.917572 30568 heartbeater.cc:499] Master 127.28.157.61:36377 was elected leader, sending a full tablet report...
14:00:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.211699 30090 heartbeater.cc:499] Master 127.28.157.61:36377 was elected leader, sending a full tablet report...
14:00:40.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:38.811618 30966 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:40.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:40.209764 30964 debug-util.cc:398] Leaking SignalData structure 0x7b0800034fe0 after lost signal to thread 30902
14:00:40.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:40.434911 30902 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.632s user 0.611s sys 0.795s
14:00:40.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:40.439441 30902 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.636s user 0.611s sys 0.795s
14:00:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.451284 30902 server_base.cc:1061] running on GCE node
14:00:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:40.452322 30997 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:00:40.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.453907 30902 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:00:40.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.463551 30902 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:00:40.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.465058 30902 hybrid_clock.cc:648] HybridClock initialized: now 1786543240464849 us; error 226 us; skew 500 ppm
14:00:40.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.466364 30902 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:00:40.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:40 dist-test-slave-vpvm krb5kdc[29316](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543240, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:40.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.474984 30902 init.cc:377] Logged in from keytab as kudu/127.28.157.60@KRBTEST.COM (short username kudu)
14:00:40.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.481496 30902 webserver.cc:533] Webserver started at http://127.28.157.60:36163/ using document root <none> and password file <none>
14:00:40.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.482817 30902 fs_manager.cc:362] Metadata directory not provided
14:00:40.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.483156 30902 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:00:40.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.495476 30902 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.007s sys 0.000s
14:00:40.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.504894 31006 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:00:40.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.508211 30902 fs_manager.cc:730] Time spent opening block manager: real 0.009s user 0.002s sys 0.004s
14:00:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.509006 30902 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/data,/tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/wal
14:00:40.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27ff31c2f960440ea7d587d161dd2a1f"
14:00:40.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 14:00:20 on dist-test-slave-vpvm"
14:00:40.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.512311 30902 fs_report.cc:389] FS layout report
14:00:40.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:00:40.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/wal
14:00:40.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/wal
14:00:40.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/data/data
14:00:40.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:00:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:00:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:00:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:00:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:00:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:00:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:00:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:00:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:00:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:00:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:00:40.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.588932 30902 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:00:40.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.596813 30902 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:00:40.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.597438 30902 kserver.cc:163] Server-wide thread pool size limit: 3276
14:00:40.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.709662 30591 master_runner.cc:187] Error getting master registration for 127.28.157.60:35027: Network error: Client connection negotiation failed: client connection to 127.28.157.60:35027: connect: Connection refused (error 111),
14:00:40.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.710233 30591 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:00:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.767172 30902 rpc_server.cc:307] RPC server started. Bound to: 127.28.157.60:35027
14:00:40.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.767266 31086 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.157.60:35027 every 8 connection(s)
14:00:40.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.772902 30902 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskTlaXut/test-tmp/mini-kudu-cluster18386158121337031790/master-2/data/info.pb
14:00:40.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.780387 29300 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu as pid 30902
14:00:40.781 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:00:40.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.824331 31088 sys_catalog.cc:263] Verifying existing consensus state
14:00:40.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.845162 31088 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f: Bootstrap starting.
14:00:40.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:40 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543238, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.2@KRBTEST.COM for kudu/127.28.157.60@KRBTEST.COM
14:00:40.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.920334 30340 heartbeater.cc:344] Connected to a master server at 127.28.157.60:35027
14:00:40.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.936781 31088 log.cc:826] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f: Log is configured to *not* fsync() on all Append() calls
14:00:40.952 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.157.61:36377(127.28.157.61:36377)] connecting to peer
14:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.973150 31088 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:40.974597 31088 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f: Bootstrap complete.
14:00:41.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.013505 30567 heartbeater.cc:344] Connected to a master server at 127.28.157.60:35027
14:00:41.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.030112 31088 raft_consensus.cc:359] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:41.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.034780 31088 raft_consensus.cc:740] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 27ff31c2f960440ea7d587d161dd2a1f, State: Initialized, Role: FOLLOWER
14:00:41.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:41 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543235, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.61@KRBTEST.COM for kudu/127.28.157.60@KRBTEST.COM
14:00:41.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.036039 31088 consensus_queue.cc:260] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } }
14:00:41.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.071487 31103 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:41.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.072382 31103 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [sys.catalog]: This master's current role is: FOLLOWER
14:00:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.076287 31088 sys_catalog.cc:565] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [sys.catalog]: configured and running, proceeding with master startup.
14:00:41.083 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.157.62:34669(127.28.157.62:34669)] connecting to peer
14:00:41.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.089736 31045 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [term 1 FOLLOWER]: Advancing to term 2
14:00:41.098 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.157.60:35027(127.28.157.60:35027)] connecting to peer
14:00:41.152 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.28.157.62:34669(127.28.157.62:34669)] Successfully connected to peer
14:00:41.154 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.28.157.60:35027(127.28.157.60:35027)] Successfully connected to peer
14:00:41.170 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.28.157.61:36377(127.28.157.61:36377)] Successfully connected to peer
14:00:41.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.220412 31103 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [sys.catalog]: SysCatalogTable state changed. Reason: New leader bab45e7c8bcd49f69477f23f98db2abf. Latest consensus state: current_term: 2 leader_uuid: "bab45e7c8bcd49f69477f23f98db2abf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:41.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.221616 31103 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [sys.catalog]: This master's current role is: FOLLOWER
14:00:41.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.227182 31127 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "bab45e7c8bcd49f69477f23f98db2abf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:41.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.228633 31127 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f [sys.catalog]: This master's current role is: FOLLOWER
14:00:41.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:41 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543235, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.1@KRBTEST.COM for kudu/127.28.157.60@KRBTEST.COM
14:00:41.320 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
14:00:41.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.369331 30089 heartbeater.cc:344] Connected to a master server at 127.28.157.60:35027
14:00:41.371 [DEBUG - kudu-nio-0] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.28.157.61
14:00:41.374 [DEBUG - kudu-nio-1] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.28.157.62
14:00:41.382 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.28.157.60
14:00:41.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.400674 31029 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "e3c457907be5422dacf0958be7c08ff7" instance_seqno: 1786543225728517) as {username='kudu', principal='kudu/127.28.157.1@KRBTEST.COM'} at 127.28.157.1:38357; Asking this server to re-register.
14:00:41.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.404457 30089 heartbeater.cc:461] Registering TS with master...
14:00:41.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.415548 31029 ts_manager.cc:194] Registered new tserver with Master: e3c457907be5422dacf0958be7c08ff7 (127.28.157.1:36855)
14:00:41.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:41 dist-test-slave-vpvm krb5kdc[29316](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543240, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.60@KRBTEST.COM for kudu/127.28.157.62@KRBTEST.COM
14:00:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.483400 31137 catalog_manager.cc:1295] Loaded cluster ID: 523c54f27a854548acd6c0ca2df51452
14:00:41.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.484798 31137 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f: loading cluster ID for follower catalog manager: success
14:00:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.516448 31137 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f: acquiring CA information for follower catalog manager: success
14:00:41.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.541265 31137 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 27ff31c2f960440ea7d587d161dd2a1f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:00:41.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.666743 30938 sys_catalog.cc:455] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "bab45e7c8bcd49f69477f23f98db2abf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "881c1b6802e24f6d9f3fb8ca5020114c" member_type: VOTER last_known_addr { host: "127.28.157.62" port: 34669 } } peers { permanent_uuid: "bab45e7c8bcd49f69477f23f98db2abf" member_type: VOTER last_known_addr { host: "127.28.157.61" port: 36377 } } peers { permanent_uuid: "27ff31c2f960440ea7d587d161dd2a1f" member_type: VOTER last_known_addr { host: "127.28.157.60" port: 35027 } } }
14:00:41.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.668285 30938 sys_catalog.cc:458] T 00000000000000000000000000000000 P bab45e7c8bcd49f69477f23f98db2abf [sys.catalog]: This master's current role is: LEADER
14:00:41.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 14:00:41 dist-test-slave-vpvm krb5kdc[29316](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786543241, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.157.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:41.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.867934 30385 init.cc:295] Successfully reacquired a new kerberos TGT
14:00:41.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.929350 31029 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2f43dc91d8564a628e22fe0c43390f9b" instance_seqno: 1786543229290644) as {username='kudu', principal='kudu/127.28.157.2@KRBTEST.COM'} at 127.28.157.2:37807; Asking this server to re-register.
14:00:41.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.932192 30340 heartbeater.cc:461] Registering TS with master...
14:00:41.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:41.937681 31029 ts_manager.cc:194] Registered new tserver with Master: 2f43dc91d8564a628e22fe0c43390f9b (127.28.157.2:36299)
14:00:42.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:42.021011 31029 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2848775df46846bb80bcba347c10f984" instance_seqno: 1786543232497729) as {username='kudu', principal='kudu/127.28.157.3@KRBTEST.COM'} at 127.28.157.3:43093; Asking this server to re-register.
14:00:42.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:42.023919 30567 heartbeater.cc:461] Registering TS with master...
14:00:42.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:42.030151 31029 ts_manager.cc:194] Registered new tserver with Master: 2848775df46846bb80bcba347c10f984 (127.28.157.3:34613)
14:00:42.181 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.157.61:36377(127.28.157.61:36377)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
14:00:42.181 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.28.157.60:35027(127.28.157.60:35027)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
14:00:42.183 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.28.157.60:35027: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
14:00:42.185 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.157.62:34669(127.28.157.62:34669)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
14:00:42.186 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.157.62:34669: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
14:00:42.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:42.189384 31098 negotiation.cc:336] Failed RPC negotiation. Trace:
14:00:42.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.140488 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:48950 (local address 127.28.157.60:35027)
14:00:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.142381 (+ 1893us) server_negotiation.cc:207] Beginning negotiation
14:00:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.142426 (+ 45us) server_negotiation.cc:400] Waiting for connection header
14:00:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.265887 (+123461us) server_negotiation.cc:408] Connection header received
14:00:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.301215 (+ 35328us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:00:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.301247 (+ 32us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:00:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.301469 (+ 222us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:00:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.301629 (+ 160us) server_negotiation.cc:227] Negotiated authn=SASL
14:00:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.925079 (+623450us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
14:00:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.926625 (+ 1546us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
14:00:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:42.144097 (+217472us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
14:00:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:42.144938 (+ 841us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
14:00:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:42.187688 (+ 42750us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:48950: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:48950): Cannot send after transport endpoint shutdown (error 108)
14:00:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":192}
14:00:42.195 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.28.157.60:35027(127.28.157.60:35027)] handling channelInactive
14:00:42.196 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.157.61:36377: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
14:00:42.200 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1412), Traces: [0ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.157.61:36377, [275ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.157.62:34669, [292ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.157.60:35027, [1376ms] Sub RPC ConnectToMaster: received response from server master-127.28.157.61:36377: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1377ms] Sub RPC ConnectToMaster: received response from server master-127.28.157.60:35027: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1380ms] Sub RPC ConnectToMaster: received response from server master-127.28.157.62:34669: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.28.157.61:36377,127.28.157.62:34669,127.28.157.60:35027). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.28.157.61:36377,127.28.157.62:34669,127.28.157.60:35027). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
14:00:42.214 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.28.157.62:34669(127.28.157.62:34669)] handling channelInactive
14:00:42.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:42.206957 31120 negotiation.cc:336] Failed RPC negotiation. Trace:
14:00:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.142602 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:48344 (local address 127.28.157.62:34669)
14:00:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.160588 (+ 17986us) server_negotiation.cc:207] Beginning negotiation
14:00:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.160618 (+ 30us) server_negotiation.cc:400] Waiting for connection header
14:00:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.265740 (+105122us) server_negotiation.cc:408] Connection header received
14:00:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.300639 (+ 34899us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:00:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.300669 (+ 30us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:00:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.300903 (+ 234us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:00:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.301249 (+ 346us) server_negotiation.cc:227] Negotiated authn=SASL
14:00:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.942321 (+641072us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
14:00:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.947739 (+ 5418us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
14:00:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:42.148839 (+201100us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
14:00:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:42.149761 (+ 922us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
14:00:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:42.205095 (+ 55334us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:48344: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:48344): Cannot send after transport endpoint shutdown (error 108)
14:00:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":16236,"thread_start_us":17154,"threads_started":1}
14:00:42.219 [DEBUG - kudu-nio-0] (KuduRpc.java:274) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1431), Traces: [0ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.157.61:36377, [275ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.157.62:34669, [292ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.157.60:35027, [1376ms] Sub RPC ConnectToMaster: received response from server master-127.28.157.61:36377: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1377ms] Sub RPC ConnectToMaster: received response from server master-127.28.157.60:35027: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1380ms] Sub RPC ConnectToMaster: received response from server master-127.28.157.62:34669: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) with no deferred attached!
14:00:42.221 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.157.60:35027(127.28.157.60:35027)] disconnecting while in state TERMINATED
14:00:42.223 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.157.62:34669(127.28.157.62:34669)] disconnecting while in state TERMINATED
14:00:42.223 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.157.61:36377(127.28.157.61:36377)] disconnecting while in state TERMINATED
14:00:42.228 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:00:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:42.229574 31047 negotiation.cc:336] Failed RPC negotiation. Trace:
14:00:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.138939 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:40838 (local address 127.28.157.61:36377)
14:00:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.140466 (+ 1527us) server_negotiation.cc:207] Beginning negotiation
14:00:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.140513 (+ 47us) server_negotiation.cc:400] Waiting for connection header
14:00:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.266443 (+125930us) server_negotiation.cc:408] Connection header received
14:00:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.300628 (+ 34185us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:00:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.300663 (+ 35us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:00:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.300919 (+ 256us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:00:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.301138 (+ 219us) server_negotiation.cc:227] Negotiated authn=SASL
14:00:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.924352 (+623214us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
14:00:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:41.926022 (+ 1670us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
14:00:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:42.147598 (+221576us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
14:00:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:42.148506 (+ 908us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
14:00:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 14:00:42.228011 (+ 79505us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:40838: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:40838): Cannot send after transport endpoint shutdown (error 108)
14:00:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":152}
14:00:42.239 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:00:42.239 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.28.157.61:36377(127.28.157.61:36377)] handling channelInactive
14:00:42.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:42.249351 29300 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu with pid 29835
14:00:42.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:42.340124 29300 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu with pid 30111
14:00:42.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:42.497123 29300 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu with pid 30365
14:00:42.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:42.552263 29300 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu with pid 30729
14:00:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:42.587831 30641 connection.cc:573] client connection to 127.28.157.62:34669 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:00:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 14:00:42.588250 30641 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107) [suppressed 7 similar messages]
14:00:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:42.589735 29300 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu with pid 30591
14:00:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:42.633801 29300 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu with pid 30902
14:00:43.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T14:00:43Z chronyd exiting
14:00:44.799 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:00:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:44.802388 28175 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu with pid 28606
14:00:44.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:44.839942 28175 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu with pid 28852
14:00:44.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:44.869300 28175 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu with pid 29066
14:00:44.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:44.921123 28175 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu with pid 28196
14:00:44.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:44.960067 28175 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu with pid 28294
14:00:45.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 14:00:45.000893 28175 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskTlaXut/build/tsan/bin/kudu with pid 28405
14:00:46.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T14:00:46Z chronyd exiting
14:00:47.125 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.28.157.61:36377,127.28.157.62:34669,127.28.157.60:35027). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.28.157.61:36377,127.28.157.62:34669,127.28.157.60:35027). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more