21:25:46.507 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:25:46.523 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskurmOAe/test-tmp
21:25:46.527 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:25:46.531 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskurmOAe/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:25:46.804 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
21:25:46.911 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:25:46.912 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:25:46.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:46.915843 28229 mini_oidc.cc:150] Starting JWKS server
21:25:46.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:46.919267 28229 webserver.cc:492] Webserver started at http://127.0.0.1:39085/ using document root <none> and password file <none>
21:25:46.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:46.919670 28229 mini_oidc.cc:186] Starting OIDC Discovery server
21:25:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:46.921447 28229 webserver.cc:492] Webserver started at http://127.0.0.1:34659/ using document root <none> and password file <none>
21:25:46.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-02-02T21:25:46Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:25:46.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-02-02T21:25:46Z Disabled control of system clock
21:25:46.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:46.981581 28229 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:25:46.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:25:46.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-0/wal
21:25:46.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-0/data
21:25:46.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:25:46.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:25:46.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:25:46.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:25:46.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:25:46.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:25:46.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-0/logs
21:25:46.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-0/data/info.pb
21:25:46.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:25:46.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:25:46.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:25:46.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:25:46.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:25:46.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:25:46.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:25:46.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:25:46.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:25:46.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:25:46.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:25:46.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:25:46.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.145.126:42349
21:25:46.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.145.126
21:25:46.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:25:46.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.145.84:33349
21:25:46.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:25:46.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:25:46.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:25:46.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:25:46.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.145.126:42349,127.27.145.125:41401,127.27.145.124:44363
21:25:46.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:25:46.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39085/jwks.json with env {}
21:25:47.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:47.278095 28241 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:25:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:47.278645 28241 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:25:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:47.279248 28241 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:25:47.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:47.310707 28241 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:25:47.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:47.310997 28241 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:25:47.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:47.311190 28241 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:25:47.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:47.311467 28241 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:25:47.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:47.311728 28241 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:25:47.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:47.311870 28241 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39085/jwks.json
21:25:47.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:47.347623 28241 master_runner.cc:386] Master server non-default flags:
21:25:47.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:25:47.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.145.84:33349
21:25:47.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:25:47.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:25:47.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-0/data
21:25:47.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-0/wal
21:25:47.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:25:47.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.145.126:42349,127.27.145.125:41401,127.27.145.124:44363
21:25:47.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:25:47.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:25:47.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:25:47.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.145.126:42349
21:25:47.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:25:47.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:25:47.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:25:47.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39085/jwks.json
21:25:47.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:25:47.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:25:47.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-0/data/info.pb
21:25:47.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.145.126
21:25:47.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:25:47.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:25:47.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:25:47.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:25:47.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:25:47.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:25:47.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-0/logs
21:25:47.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:25:47.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:25:47.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:25:47.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:25:47.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:25:47.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:25:47.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:25:47.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:25:47.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:25:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:25:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:47.349100 28241 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:25:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:47.350908 28241 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:25:47.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:47.363160 28247 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:25:48.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:47.363811 28248 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:25:48.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:48.572886 28250 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:25:48.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:48.575917 28249 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1206 milliseconds
21:25:48.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:48.576417 28241 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.214s user 0.368s sys 0.836s
21:25:48.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:48.576785 28241 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.214s user 0.368s sys 0.836s
21:25:48.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.577093 28241 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:25:48.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.578727 28241 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:25:48.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.582171 28241 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:25:48.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.583938 28241 hybrid_clock.cc:648] HybridClock initialized: now 1770067548583895 us; error 32 us; skew 500 ppm
21:25:48.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.585175 28241 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:25:48.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.594136 28241 webserver.cc:492] Webserver started at http://127.27.145.126:46873/ using document root <none> and password file <none>
21:25:48.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.595569 28241 fs_manager.cc:362] Metadata directory not provided
21:25:48.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.595875 28241 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:25:48.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.596505 28241 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:25:48.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.603509 28241 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-0/data/instance:
21:25:48.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d95a26c2d3bb4309824eefae23a709c9"
21:25:48.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:25:48 on dist-test-slave-3trw"
21:25:48.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.605115 28241 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-0/wal/instance:
21:25:48.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d95a26c2d3bb4309824eefae23a709c9"
21:25:48.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:25:48 on dist-test-slave-3trw"
21:25:48.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.615097 28241 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.006s sys 0.002s
21:25:48.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.623111 28257 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:25:48.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.624457 28241 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
21:25:48.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.624883 28241 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-0/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-0/wal
21:25:48.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d95a26c2d3bb4309824eefae23a709c9"
21:25:48.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:25:48 on dist-test-slave-3trw"
21:25:48.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.625353 28241 fs_report.cc:389] FS layout report
21:25:48.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:25:48.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-0/wal
21:25:48.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-0/wal
21:25:48.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-0/data/data
21:25:48.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:25:48.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:25:48.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:25:48.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:25:48.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:25:48.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:25:48.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:25:48.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:25:48.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:25:48.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:25:48.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:25:48.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.713718 28241 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:25:48.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.715199 28241 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:25:48.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.715597 28241 kserver.cc:163] Server-wide thread pool size limit: 3276
21:25:48.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.784245 28241 rpc_server.cc:307] RPC server started. Bound to: 127.27.145.126:42349
21:25:48.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.784315 28308 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.145.126:42349 every 8 connection(s)
21:25:48.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.787163 28241 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-0/data/info.pb
21:25:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.792997 28309 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:25:48.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.796800 28229 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 28241
21:25:48.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.797279 28229 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-0/wal/instance
21:25:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.800669 28229 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:25:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:25:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-1/wal
21:25:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-1/data
21:25:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:25:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:25:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:25:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:25:48.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:25:48.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:25:48.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-1/logs
21:25:48.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-1/data/info.pb
21:25:48.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:25:48.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:25:48.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:25:48.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:25:48.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:25:48.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:25:48.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:25:48.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:25:48.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:25:48.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:25:48.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:25:48.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:25:48.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.145.125:41401
21:25:48.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.145.125
21:25:48.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:25:48.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.145.84:33349
21:25:48.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:25:48.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:25:48.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:25:48.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:25:48.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.145.126:42349,127.27.145.125:41401,127.27.145.124:44363
21:25:48.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:25:48.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39085/jwks.json with env {}
21:25:48.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.803395 28309 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } has no permanent_uuid. Determining permanent_uuid...
21:25:48.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.832700 28309 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } has no permanent_uuid. Determining permanent_uuid...
21:25:48.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:48.835111 28260 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.145.125:41401: connect: Connection refused (error 111)
21:25:48.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:48.839349 28309 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.125:41401: Network error: Client connection negotiation failed: client connection to 127.27.145.125:41401: connect: Connection refused (error 111)
21:25:48.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.888830 28309 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } attempt: 1
21:25:48.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:48.894061 28309 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.125:41401: Network error: Client connection negotiation failed: client connection to 127.27.145.125:41401: connect: Connection refused (error 111)
21:25:48.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:48.962469 28309 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } attempt: 2
21:25:48.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:48.966771 28309 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.125:41401: Network error: Client connection negotiation failed: client connection to 127.27.145.125:41401: connect: Connection refused (error 111)
21:25:49.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:49.058118 28309 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } attempt: 3
21:25:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:49.062438 28309 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.125:41401: Network error: Client connection negotiation failed: client connection to 127.27.145.125:41401: connect: Connection refused (error 111)
21:25:49.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:49.109632 28311 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:25:49.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:49.110433 28311 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:25:49.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:49.111122 28311 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:25:49.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:49.142529 28311 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:25:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:49.142913 28311 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:25:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:49.143252 28311 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:25:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:49.143652 28311 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:25:49.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:49.144013 28311 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:25:49.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:49.144275 28311 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39085/jwks.json
21:25:49.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:49.180473 28311 master_runner.cc:386] Master server non-default flags:
21:25:49.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:25:49.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.145.84:33349
21:25:49.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:25:49.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:25:49.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-1/data
21:25:49.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-1/wal
21:25:49.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:25:49.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.145.126:42349,127.27.145.125:41401,127.27.145.124:44363
21:25:49.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:25:49.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:25:49.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:25:49.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.145.125:41401
21:25:49.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:25:49.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:25:49.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:25:49.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39085/jwks.json
21:25:49.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:25:49.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:25:49.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-1/data/info.pb
21:25:49.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.145.125
21:25:49.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:25:49.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:25:49.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:25:49.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:25:49.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:25:49.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:25:49.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-1/logs
21:25:49.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:25:49.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:25:49.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:25:49.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:25:49.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:25:49.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:25:49.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:25:49.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:25:49.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:25:49.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:25:49.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:49.182247 28311 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:25:49.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:49.184404 28311 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:25:49.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:49.195492 28320 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:25:49.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:49.205863 28309 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } attempt: 4
21:25:49.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:49.212321 28309 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.125:41401: Network error: Client connection negotiation failed: client connection to 127.27.145.125:41401: connect: Connection refused (error 111)
21:25:49.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:49.511811 28309 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } attempt: 5
21:25:49.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:49.520233 28309 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.125:41401: Network error: Client connection negotiation failed: client connection to 127.27.145.125:41401: connect: Connection refused (error 111)
21:25:50.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.067828 28309 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } attempt: 6
21:25:50.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:50.095535 28309 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.125:41401: Network error: Client connection negotiation failed: client connection to 127.27.145.125:41401: connect: Connection refused (error 111)
21:25:50.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:50.600469 28319 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec80 after lost signal to thread 28311
21:25:50.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:50.644984 28319 kernel_stack_watchdog.cc:198] Thread 28311 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
21:25:50.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:25:50.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:25:50.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:25:50.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:25:50.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:50.645545 28311 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.450s user 0.477s sys 0.965s
21:25:50.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:50.645926 28311 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.451s user 0.477s sys 0.966s
21:25:50.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:49.196022 28321 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:25:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:50.646687 28322 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1449 milliseconds
21:25:50.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:50.648571 28323 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:25:50.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.648497 28311 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:25:50.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.652099 28311 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:25:50.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.654579 28311 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:25:50.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.655916 28311 hybrid_clock.cc:648] HybridClock initialized: now 1770067550655893 us; error 52 us; skew 500 ppm
21:25:50.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.656704 28311 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:25:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.662909 28311 webserver.cc:492] Webserver started at http://127.27.145.125:38665/ using document root <none> and password file <none>
21:25:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.663808 28311 fs_manager.cc:362] Metadata directory not provided
21:25:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.664012 28311 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:25:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.664397 28311 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:25:50.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.668664 28311 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-1/data/instance:
21:25:50.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "038b8a8c2be446baa8ff9bfc1b7a6278"
21:25:50.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:25:50 on dist-test-slave-3trw"
21:25:50.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.669795 28311 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-1/wal/instance:
21:25:50.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "038b8a8c2be446baa8ff9bfc1b7a6278"
21:25:50.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:25:50 on dist-test-slave-3trw"
21:25:50.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.676878 28311 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.001s
21:25:50.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.682660 28332 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:25:50.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.683787 28311 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
21:25:50.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.684057 28311 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-1/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-1/wal
21:25:50.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "038b8a8c2be446baa8ff9bfc1b7a6278"
21:25:50.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:25:50 on dist-test-slave-3trw"
21:25:50.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.684343 28311 fs_report.cc:389] FS layout report
21:25:50.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:25:50.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-1/wal
21:25:50.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-1/wal
21:25:50.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-1/data/data
21:25:50.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:25:50.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:25:50.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:25:50.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:25:50.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:25:50.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:25:50.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:25:50.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:25:50.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:25:50.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:25:50.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:25:50.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.732316 28311 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:25:50.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.733765 28311 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:25:50.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.734239 28311 kserver.cc:163] Server-wide thread pool size limit: 3276
21:25:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.806344 28311 rpc_server.cc:307] RPC server started. Bound to: 127.27.145.125:41401
21:25:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.806416 28383 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.145.125:41401 every 8 connection(s)
21:25:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.809263 28311 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-1/data/info.pb
21:25:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.810343 28229 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 28311
21:25:50.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.810796 28229 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-1/wal/instance
21:25:50.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.813520 28229 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:25:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:25:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-2/wal
21:25:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-2/data
21:25:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:25:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:25:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:25:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:25:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:25:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:25:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-2/logs
21:25:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-2/data/info.pb
21:25:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:25:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:25:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:25:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:25:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:25:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:25:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:25:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:25:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:25:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:25:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:25:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:25:50.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.145.124:44363
21:25:50.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.145.124
21:25:50.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:25:50.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.145.84:33349
21:25:50.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:25:50.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:25:50.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:25:50.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:25:50.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.145.126:42349,127.27.145.125:41401,127.27.145.124:44363
21:25:50.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:25:50.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39085/jwks.json with env {}
21:25:50.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.816411 28384 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:25:50.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.828044 28384 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } has no permanent_uuid. Determining permanent_uuid...
21:25:50.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.851666 28384 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } has no permanent_uuid. Determining permanent_uuid...
21:25:50.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.866003 28384 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } has no permanent_uuid. Determining permanent_uuid...
21:25:50.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:50.868628 28336 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.145.124:44363: connect: Connection refused (error 111)
21:25:50.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:50.872990 28384 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.124:44363: Network error: Client connection negotiation failed: client connection to 127.27.145.124:44363: connect: Connection refused (error 111)
21:25:50.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.922451 28384 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } attempt: 1
21:25:50.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:50.926684 28384 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.124:44363: Network error: Client connection negotiation failed: client connection to 127.27.145.124:44363: connect: Connection refused (error 111)
21:25:50.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:50.995050 28384 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } attempt: 2
21:25:50.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:50.999320 28384 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.124:44363: Network error: Client connection negotiation failed: client connection to 127.27.145.124:44363: connect: Connection refused (error 111)
21:25:51.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:51.090763 28384 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } attempt: 3
21:25:51.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.095456 28384 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.124:44363: Network error: Client connection negotiation failed: client connection to 127.27.145.124:44363: connect: Connection refused (error 111)
21:25:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.126591 28386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:25:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.127257 28386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:25:51.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.127928 28386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:25:51.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:51.156162 28309 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } attempt: 7
21:25:51.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.160830 28386 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:25:51.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.161185 28386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:25:51.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.161571 28386 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:25:51.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.161909 28386 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:25:51.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.162463 28386 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:25:51.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.162977 28386 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39085/jwks.json
21:25:51.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:51.168080 28309 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } has no permanent_uuid. Determining permanent_uuid...
21:25:51.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.173657 28309 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.124:44363: Network error: Client connection negotiation failed: client connection to 127.27.145.124:44363: connect: Connection refused (error 111)
21:25:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:51.203691 28386 master_runner.cc:386] Master server non-default flags:
21:25:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:25:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.145.84:33349
21:25:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:25:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:25:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-2/data
21:25:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-2/wal
21:25:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:25:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.145.126:42349,127.27.145.125:41401,127.27.145.124:44363
21:25:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:25:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:25:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:25:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.145.124:44363
21:25:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:25:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:25:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:25:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39085/jwks.json
21:25:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:25:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:25:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-2/data/info.pb
21:25:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.145.124
21:25:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:25:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:25:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:25:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:25:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:25:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:25:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-2/logs
21:25:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:25:51.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:25:51.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:25:51.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:25:51.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:25:51.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:25:51.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:25:51.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:25:51.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:25:51.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:25:51.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:51.205004 28386 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:25:51.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:51.207163 28386 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:25:51.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.220466 28398 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:25:51.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:51.232091 28309 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } attempt: 1
21:25:51.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.237954 28309 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.124:44363: Network error: Client connection negotiation failed: client connection to 127.27.145.124:44363: connect: Connection refused (error 111)
21:25:51.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:51.238826 28384 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } attempt: 4
21:25:51.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.247144 28384 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.124:44363: Network error: Client connection negotiation failed: client connection to 127.27.145.124:44363: connect: Connection refused (error 111)
21:25:51.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:51.319406 28309 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } attempt: 2
21:25:51.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.327064 28309 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.124:44363: Network error: Client connection negotiation failed: client connection to 127.27.145.124:44363: connect: Connection refused (error 111)
21:25:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:51.412683 28309 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } attempt: 3
21:25:51.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.426327 28309 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.124:44363: Network error: Client connection negotiation failed: client connection to 127.27.145.124:44363: connect: Connection refused (error 111)
21:25:51.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:51.546680 28384 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } attempt: 5
21:25:51.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.554230 28384 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.124:44363: Network error: Client connection negotiation failed: client connection to 127.27.145.124:44363: connect: Connection refused (error 111)
21:25:51.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:51.566890 28309 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } attempt: 4
21:25:51.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.574107 28309 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.124:44363: Network error: Client connection negotiation failed: client connection to 127.27.145.124:44363: connect: Connection refused (error 111)
21:25:51.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:51.857646 28309 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } attempt: 5
21:25:51.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.865881 28309 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.124:44363: Network error: Client connection negotiation failed: client connection to 127.27.145.124:44363: connect: Connection refused (error 111)
21:25:52.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.101814 28384 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } attempt: 6
21:25:52.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:52.132390 28384 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.124:44363: Network error: Client connection negotiation failed: client connection to 127.27.145.124:44363: connect: Connection refused (error 111)
21:25:52.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.418490 28309 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } attempt: 6
21:25:52.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:52.445269 28309 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.145.124:44363: Network error: Client connection negotiation failed: client connection to 127.27.145.124:44363: connect: Connection refused (error 111)
21:25:52.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:52.625633 28397 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec80 after lost signal to thread 28386
21:25:52.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:52.760517 28386 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.540s user 0.555s sys 0.961s
21:25:52.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:52.760986 28386 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.540s user 0.555s sys 0.962s
21:25:52.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:51.222170 28399 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:25:52.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:52.762028 28400 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1539 milliseconds
21:25:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.763362 28386 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:25:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:52.763437 28401 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:25:52.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.766883 28386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:25:52.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.769269 28386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:25:52.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.770596 28386 hybrid_clock.cc:648] HybridClock initialized: now 1770067552770562 us; error 58 us; skew 500 ppm
21:25:52.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.771371 28386 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:25:52.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.777338 28386 webserver.cc:492] Webserver started at http://127.27.145.124:32923/ using document root <none> and password file <none>
21:25:52.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.778285 28386 fs_manager.cc:362] Metadata directory not provided
21:25:52.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.778486 28386 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:25:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.778856 28386 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:25:52.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.786249 28386 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-2/data/instance:
21:25:52.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8ae02a7cf87749f2abfe516d8db7864a"
21:25:52.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:25:52 on dist-test-slave-3trw"
21:25:52.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.787325 28386 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-2/wal/instance:
21:25:52.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8ae02a7cf87749f2abfe516d8db7864a"
21:25:52.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:25:52 on dist-test-slave-3trw"
21:25:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.794416 28386 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.001s sys 0.006s
21:25:52.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.799914 28412 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:25:52.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.801137 28386 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.005s
21:25:52.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.801525 28386 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-2/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-2/wal
21:25:52.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8ae02a7cf87749f2abfe516d8db7864a"
21:25:52.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:25:52 on dist-test-slave-3trw"
21:25:52.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.801964 28386 fs_report.cc:389] FS layout report
21:25:52.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:25:52.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-2/wal
21:25:52.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-2/wal
21:25:52.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-2/data/data
21:25:52.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:25:52.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:25:52.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:25:52.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:25:52.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:25:52.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:25:52.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:25:52.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:25:52.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:25:52.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:25:52.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:25:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.864609 28386 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:25:52.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.866122 28386 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:25:52.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.866539 28386 kserver.cc:163] Server-wide thread pool size limit: 3276
21:25:52.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.934984 28386 rpc_server.cc:307] RPC server started. Bound to: 127.27.145.124:44363
21:25:52.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.935050 28463 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.145.124:44363 every 8 connection(s)
21:25:52.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.937752 28386 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-2/data/info.pb
21:25:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.941879 28229 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 28386
21:25:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.942356 28229 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/master-2/wal/instance
21:25:52.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.944669 28464 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:25:52.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.961664 28464 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } has no permanent_uuid. Determining permanent_uuid...
21:25:52.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:52.994313 28464 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } has no permanent_uuid. Determining permanent_uuid...
21:25:53.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.004401 28464 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } has no permanent_uuid. Determining permanent_uuid...
21:25:53.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.020078 28464 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a: Bootstrap starting.
21:25:53.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.026311 28464 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a: Neither blocks nor log segments found. Creating new log.
21:25:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.027887 28464 log.cc:826] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a: Log is configured to *not* fsync() on all Append() calls
21:25:53.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.031849 28464 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a: No bootstrap required, opened a new log
21:25:53.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.050602 28464 raft_consensus.cc:359] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } }
21:25:53.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.051214 28464 raft_consensus.cc:385] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:25:53.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.051402 28464 raft_consensus.cc:740] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8ae02a7cf87749f2abfe516d8db7864a, State: Initialized, Role: FOLLOWER
21:25:53.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.052219 28464 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } }
21:25:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.054448 28473 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } } }
21:25:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.055284 28473 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [sys.catalog]: This master's current role is: FOLLOWER
21:25:53.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.056151 28464 sys_catalog.cc:565] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [sys.catalog]: configured and running, proceeding with master startup.
21:25:53.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:53.079456 28484 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:25:53.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:53.079732 28484 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:25:53.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.076001 28386 master_runner.cc:186] Error getting master registration for 127.27.145.126:42349: OK, instance_id { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" instance_seqno: 1770067548765246 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:25:53.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.100360 28386 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:25:53.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:53.121503 28418 tablet.cc:2378] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:25:53.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.192924 28384 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } attempt: 7
21:25:53.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.211382 28384 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278: Bootstrap starting.
21:25:53.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.216729 28384 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278: Neither blocks nor log segments found. Creating new log.
21:25:53.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.218843 28384 log.cc:826] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278: Log is configured to *not* fsync() on all Append() calls
21:25:53.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.224001 28384 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278: No bootstrap required, opened a new log
21:25:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.242090 28384 raft_consensus.cc:359] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } }
21:25:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.243322 28384 raft_consensus.cc:385] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:25:53.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.243845 28384 raft_consensus.cc:740] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 038b8a8c2be446baa8ff9bfc1b7a6278, State: Initialized, Role: FOLLOWER
21:25:53.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.244750 28384 consensus_queue.cc:260] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } }
21:25:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.247547 28488 sys_catalog.cc:455] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } } }
21:25:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.248346 28488 sys_catalog.cc:458] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278 [sys.catalog]: This master's current role is: FOLLOWER
21:25:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.249387 28384 sys_catalog.cc:565] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278 [sys.catalog]: configured and running, proceeding with master startup.
21:25:53.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:53.281280 28499 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:25:53.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:53.282146 28499 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:25:53.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.276291 28311 master_runner.cc:186] Error getting master registration for 127.27.145.126:42349: OK, instance_id { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" instance_seqno: 1770067548765246 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:25:53.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.308971 28311 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:25:53.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.388186 28473 raft_consensus.cc:493] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:25:53.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.388754 28473 raft_consensus.cc:515] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } }
21:25:53.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.400698 28473 leader_election.cc:290] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d95a26c2d3bb4309824eefae23a709c9 (127.27.145.126:42349), 038b8a8c2be446baa8ff9bfc1b7a6278 (127.27.145.125:41401)
21:25:53.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.400779 28359 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8ae02a7cf87749f2abfe516d8db7864a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" is_pre_election: true
21:25:53.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.401734 28359 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8ae02a7cf87749f2abfe516d8db7864a in term 0.
21:25:53.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.403280 28415 leader_election.cc:304] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 038b8a8c2be446baa8ff9bfc1b7a6278, 8ae02a7cf87749f2abfe516d8db7864a; no voters:
21:25:53.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.404220 28473 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [term 0 FOLLOWER]: Leader pre-election won for term 1
21:25:53.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.404569 28473 raft_consensus.cc:493] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:25:53.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.404897 28473 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [term 0 FOLLOWER]: Advancing to term 1
21:25:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.411710 28473 raft_consensus.cc:515] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } }
21:25:53.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.414570 28359 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8ae02a7cf87749f2abfe516d8db7864a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278"
21:25:53.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.415112 28359 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278 [term 0 FOLLOWER]: Advancing to term 1
21:25:53.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.420575 28473 leader_election.cc:290] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [CANDIDATE]: Term 1 election: Requested vote from peers d95a26c2d3bb4309824eefae23a709c9 (127.27.145.126:42349), 038b8a8c2be446baa8ff9bfc1b7a6278 (127.27.145.125:41401)
21:25:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.422935 28359 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8ae02a7cf87749f2abfe516d8db7864a in term 1.
21:25:53.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.424163 28415 leader_election.cc:304] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 038b8a8c2be446baa8ff9bfc1b7a6278, 8ae02a7cf87749f2abfe516d8db7864a; no voters:
21:25:53.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.425029 28473 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [term 1 FOLLOWER]: Leader election won for term 1
21:25:53.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.426954 28473 raft_consensus.cc:697] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [term 1 LEADER]: Becoming Leader. State: Replica: 8ae02a7cf87749f2abfe516d8db7864a, State: Running, Role: LEADER
21:25:53.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.428010 28473 consensus_queue.cc:237] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } }
21:25:53.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.439066 28501 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8ae02a7cf87749f2abfe516d8db7864a. Latest consensus state: current_term: 1 leader_uuid: "8ae02a7cf87749f2abfe516d8db7864a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } } }
21:25:53.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.440016 28501 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [sys.catalog]: This master's current role is: LEADER
21:25:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.443168 28503 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:25:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.453810 28503 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:25:53.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.478860 28309 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } attempt: 7
21:25:53.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.491554 28359 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278 [term 1 FOLLOWER]: Refusing update from remote peer 8ae02a7cf87749f2abfe516d8db7864a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:25:53.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:53.491850 28338 tablet.cc:2378] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:25:53.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.493862 28501 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [LEADER]: Connected to new peer: Peer: permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:25:53.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.521312 28309 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9: Bootstrap starting.
21:25:53.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.529277 28309 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9: Neither blocks nor log segments found. Creating new log.
21:25:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.532001 28309 log.cc:826] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9: Log is configured to *not* fsync() on all Append() calls
21:25:53.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.539477 28488 sys_catalog.cc:455] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8ae02a7cf87749f2abfe516d8db7864a. Latest consensus state: current_term: 1 leader_uuid: "8ae02a7cf87749f2abfe516d8db7864a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } } }
21:25:53.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.540191 28488 sys_catalog.cc:458] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278 [sys.catalog]: This master's current role is: FOLLOWER
21:25:53.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.549711 28473 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8ae02a7cf87749f2abfe516d8db7864a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } } }
21:25:53.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.550469 28473 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [sys.catalog]: This master's current role is: LEADER
21:25:53.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.554666 28503 catalog_manager.cc:1357] Generated new cluster ID: 36c38aa9fa86496d9bcc333293a1bbc4
21:25:53.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.554965 28503 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:25:53.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.555967 28488 sys_catalog.cc:455] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8ae02a7cf87749f2abfe516d8db7864a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } } }
21:25:53.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.556617 28488 sys_catalog.cc:458] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278 [sys.catalog]: This master's current role is: FOLLOWER
21:25:53.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.574601 28309 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9: No bootstrap required, opened a new log
21:25:53.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.619756 28503 catalog_manager.cc:1380] Generated new certificate authority record
21:25:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.621816 28503 catalog_manager.cc:1514] Loading token signing keys...
21:25:53.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.641892 28309 raft_consensus.cc:359] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } }
21:25:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.644531 28309 raft_consensus.cc:385] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:25:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.645607 28309 raft_consensus.cc:740] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d95a26c2d3bb4309824eefae23a709c9, State: Initialized, Role: FOLLOWER
21:25:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.648229 28309 consensus_queue.cc:260] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } }
21:25:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.668800 28503 catalog_manager.cc:6027] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a: Generated new TSK 0
21:25:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.669888 28503 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:25:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.664126 28511 sys_catalog.cc:455] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } } }
21:25:53.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.671008 28511 sys_catalog.cc:458] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9 [sys.catalog]: This master's current role is: FOLLOWER
21:25:53.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.672384 28284 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8ae02a7cf87749f2abfe516d8db7864a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d95a26c2d3bb4309824eefae23a709c9" is_pre_election: true
21:25:53.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.673209 28284 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8ae02a7cf87749f2abfe516d8db7864a in term 0.
21:25:53.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.674147 28283 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8ae02a7cf87749f2abfe516d8db7864a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d95a26c2d3bb4309824eefae23a709c9"
21:25:53.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.674695 28283 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9 [term 0 FOLLOWER]: Advancing to term 1
21:25:53.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.678609 28309 sys_catalog.cc:565] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9 [sys.catalog]: configured and running, proceeding with master startup.
21:25:53.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.681371 28283 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8ae02a7cf87749f2abfe516d8db7864a in term 1.
21:25:53.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.682741 28282 raft_consensus.cc:1275] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9 [term 1 FOLLOWER]: Refusing update from remote peer 8ae02a7cf87749f2abfe516d8db7864a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:25:53.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.690361 28501 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [LEADER]: Connected to new peer: Peer: permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:25:53.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.691453 28229 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:25:53.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:25:53.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-0/wal
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-0/data
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-0/logs
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-0/data/info.pb
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:25:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.145.65:0
21:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.145.65
21:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.145.65
21:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.145.126:42349,127.27.145.125:41401,127.27.145.124:44363
21:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.145.84:33349
21:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:25:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:25:53.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:53.725813 28263 tablet.cc:2378] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:25:53.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.762908 28511 sys_catalog.cc:455] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8ae02a7cf87749f2abfe516d8db7864a. Latest consensus state: current_term: 1 leader_uuid: "8ae02a7cf87749f2abfe516d8db7864a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } } }
21:25:53.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.764063 28511 sys_catalog.cc:458] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9 [sys.catalog]: This master's current role is: FOLLOWER
21:25:53.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.763165 28520 sys_catalog.cc:455] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8ae02a7cf87749f2abfe516d8db7864a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } } }
21:25:53.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.765398 28520 sys_catalog.cc:458] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9 [sys.catalog]: This master's current role is: FOLLOWER
21:25:53.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:53.804936 28529 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:25:54.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:54.071321 28501 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8ae02a7cf87749f2abfe516d8db7864a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d95a26c2d3bb4309824eefae23a709c9" member_type: VOTER last_known_addr { host: "127.27.145.126" port: 42349 } } peers { permanent_uuid: "038b8a8c2be446baa8ff9bfc1b7a6278" member_type: VOTER last_known_addr { host: "127.27.145.125" port: 41401 } } peers { permanent_uuid: "8ae02a7cf87749f2abfe516d8db7864a" member_type: VOTER last_known_addr { host: "127.27.145.124" port: 44363 } } }
21:25:54.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:54.072193 28501 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8ae02a7cf87749f2abfe516d8db7864a [sys.catalog]: This master's current role is: LEADER
21:25:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:54.230357 28515 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:25:54.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:54.230916 28515 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:25:54.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:54.231560 28515 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:25:54.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:54.265923 28515 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:25:54.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:54.267874 28515 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.145.65
21:25:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:54.292474 28499 catalog_manager.cc:1269] Loaded cluster ID: 36c38aa9fa86496d9bcc333293a1bbc4
21:25:54.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:54.293085 28499 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278: loading cluster ID for follower catalog manager: success
21:25:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:54.300550 28499 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278: acquiring CA information for follower catalog manager: success
21:25:54.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:54.306016 28499 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 038b8a8c2be446baa8ff9bfc1b7a6278: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:25:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:54.328771 28515 tablet_server_runner.cc:78] Tablet server non-default flags:
21:25:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:25:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.145.84:33349
21:25:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:25:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:25:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-0/data
21:25:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-0/wal
21:25:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:25:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:25:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.145.65:0
21:25:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:25:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-0/data/info.pb
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.145.65
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.145.126:42349,127.27.145.125:41401,127.27.145.124:44363
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.145.65
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-0/logs
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:25:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:25:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:25:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:25:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:25:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:54.330485 28515 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:25:54.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:54.332508 28515 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:25:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:54.349711 28536 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:25:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:54.838972 28529 catalog_manager.cc:1269] Loaded cluster ID: 36c38aa9fa86496d9bcc333293a1bbc4
21:25:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:54.839313 28529 catalog_manager.cc:1562] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9: loading cluster ID for follower catalog manager: success
21:25:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:54.848165 28529 catalog_manager.cc:1584] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9: acquiring CA information for follower catalog manager: success
21:25:54.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:54.870374 28529 catalog_manager.cc:1612] T 00000000000000000000000000000000 P d95a26c2d3bb4309824eefae23a709c9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:25:56.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:55.750118 28535 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 28515
21:25:56.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:56.983445 28535 kernel_stack_watchdog.cc:198] Thread 28515 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 395ms:
21:25:56.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:25:56.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:25:56.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:25:56.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:25:56.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:56.986297 28539 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:25:56.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:56.987480 28515 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.635s user 0.739s sys 1.312s
21:25:56.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:56.988013 28515 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.636s user 0.740s sys 1.312s
21:25:56.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:54.352744 28537 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:25:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:56.994807 28538 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2633 milliseconds
21:25:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:56.995571 28515 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:25:56.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:56.996996 28515 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:25:57.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.000036 28515 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:25:57.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.001499 28515 hybrid_clock.cc:648] HybridClock initialized: now 1770067557001349 us; error 161 us; skew 500 ppm
21:25:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.002621 28515 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:25:57.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.013959 28515 webserver.cc:492] Webserver started at http://127.27.145.65:40451/ using document root <none> and password file <none>
21:25:57.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.015228 28515 fs_manager.cc:362] Metadata directory not provided
21:25:57.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.015537 28515 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:25:57.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.016099 28515 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:25:57.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.022233 28515 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-0/data/instance:
21:25:57.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cf78553c80604aacb392c81092398e43"
21:25:57.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:25:57 on dist-test-slave-3trw"
21:25:57.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.023703 28515 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-0/wal/instance:
21:25:57.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cf78553c80604aacb392c81092398e43"
21:25:57.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:25:57 on dist-test-slave-3trw"
21:25:57.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.033713 28515 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.010s sys 0.000s
21:25:57.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.041273 28548 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:25:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.042729 28515 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
21:25:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.043109 28515 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-0/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-0/wal
21:25:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cf78553c80604aacb392c81092398e43"
21:25:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:25:57 on dist-test-slave-3trw"
21:25:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.043514 28515 fs_report.cc:389] FS layout report
21:25:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:25:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-0/wal
21:25:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-0/wal
21:25:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-0/data/data
21:25:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:25:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:25:57.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:25:57.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:25:57.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:25:57.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:25:57.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:25:57.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:25:57.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:25:57.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:25:57.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:25:57.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.124096 28515 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:25:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.126133 28515 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:25:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.126690 28515 kserver.cc:163] Server-wide thread pool size limit: 3276
21:25:57.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.130198 28515 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:25:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.136420 28515 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:25:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.136729 28515 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:25:57.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.137028 28515 ts_tablet_manager.cc:616] Registered 0 tablets
21:25:57.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.137238 28515 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:25:57.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.323632 28515 rpc_server.cc:307] RPC server started. Bound to: 127.27.145.65:44013
21:25:57.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.323700 28661 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.145.65:44013 every 8 connection(s)
21:25:57.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.327339 28515 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-0/data/info.pb
21:25:57.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.336998 28229 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 28515
21:25:57.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.337776 28229 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-0/wal/instance
21:25:57.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.358083 28229 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:25:57.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:25:57.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-1/wal
21:25:57.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-1/data
21:25:57.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:25:57.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:25:57.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:25:57.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:25:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:25:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:25:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-1/logs
21:25:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-1/data/info.pb
21:25:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:25:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:25:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:25:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:25:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:25:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:25:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:25:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:25:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:25:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:25:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.145.66:0
21:25:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.145.66
21:25:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.145.66
21:25:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:25:57.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.145.126:42349,127.27.145.125:41401,127.27.145.124:44363
21:25:57.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.145.84:33349
21:25:57.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:25:57.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:25:57.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:25:57.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.402473 28663 heartbeater.cc:344] Connected to a master server at 127.27.145.125:41401
21:25:57.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.403044 28663 heartbeater.cc:461] Registering TS with master...
21:25:57.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.404865 28663 heartbeater.cc:507] Master 127.27.145.125:41401 requested a full tablet report, sending...
21:25:57.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.408704 28349 ts_manager.cc:194] Registered new tserver with Master: cf78553c80604aacb392c81092398e43 (127.27.145.65:44013)
21:25:57.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.409310 28664 heartbeater.cc:344] Connected to a master server at 127.27.145.126:42349
21:25:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.409646 28664 heartbeater.cc:461] Registering TS with master...
21:25:57.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.410563 28664 heartbeater.cc:507] Master 127.27.145.126:42349 requested a full tablet report, sending...
21:25:57.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.410605 28662 heartbeater.cc:344] Connected to a master server at 127.27.145.124:44363
21:25:57.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.411157 28662 heartbeater.cc:461] Registering TS with master...
21:25:57.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.412544 28662 heartbeater.cc:507] Master 127.27.145.124:44363 requested a full tablet report, sending...
21:25:57.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.414398 28274 ts_manager.cc:194] Registered new tserver with Master: cf78553c80604aacb392c81092398e43 (127.27.145.65:44013)
21:25:57.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.415058 28429 ts_manager.cc:194] Registered new tserver with Master: cf78553c80604aacb392c81092398e43 (127.27.145.65:44013)
21:25:57.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.417725 28429 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.27.145.65:40391
21:25:57.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:57.726948 28672 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:25:57.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:57.727401 28672 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:25:57.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:57.727802 28672 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:25:57.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:57.759788 28672 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:25:57.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:57.760874 28672 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.145.66
21:25:57.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.816857 28672 tablet_server_runner.cc:78] Tablet server non-default flags:
21:25:57.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:25:57.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.145.84:33349
21:25:57.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:25:57.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:25:57.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-1/data
21:25:57.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-1/wal
21:25:57.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:25:57.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.145.66:0
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-1/data/info.pb
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.145.66
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.145.126:42349,127.27.145.125:41401,127.27.145.124:44363
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.145.66
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-1/logs
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:25:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:25:57.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.820225 28672 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:25:57.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:57.822458 28672 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:25:57.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:57.840816 28679 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:25:58.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:58.421706 28662 heartbeater.cc:499] Master 127.27.145.124:44363 was elected leader, sending a full tablet report...
21:25:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:59.242810 28677 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 28672
21:25:59.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:59.356279 28677 kernel_stack_watchdog.cc:198] Thread 28672 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
21:25:59.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:25:59.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:25:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:25:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:25:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:59.357035 28672 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.516s user 0.544s sys 0.879s
21:25:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:59.357416 28672 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.516s user 0.544s sys 0.879s
21:25:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:57.841027 28678 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:25:59.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:59.359452 28681 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:25:59.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:25:59.362886 28680 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1517 milliseconds
21:25:59.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.362943 28672 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:25:59.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.364197 28672 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:25:59.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.366392 28672 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:25:59.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.367772 28672 hybrid_clock.cc:648] HybridClock initialized: now 1770067559367737 us; error 36 us; skew 500 ppm
21:25:59.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.369029 28672 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:25:59.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.377974 28672 webserver.cc:492] Webserver started at http://127.27.145.66:36233/ using document root <none> and password file <none>
21:25:59.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.379408 28672 fs_manager.cc:362] Metadata directory not provided
21:25:59.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.379652 28672 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:25:59.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.380472 28672 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:25:59.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.386164 28672 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-1/data/instance:
21:25:59.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "50e9ad1f1f4a4ed0b4c8a8bea5224b20"
21:25:59.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:25:59 on dist-test-slave-3trw"
21:25:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.387737 28672 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-1/wal/instance:
21:25:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "50e9ad1f1f4a4ed0b4c8a8bea5224b20"
21:25:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:25:59 on dist-test-slave-3trw"
21:25:59.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.397831 28672 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.009s sys 0.001s
21:25:59.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.406235 28689 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:25:59.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.407548 28672 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
21:25:59.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.407893 28672 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-1/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-1/wal
21:25:59.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "50e9ad1f1f4a4ed0b4c8a8bea5224b20"
21:25:59.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:25:59 on dist-test-slave-3trw"
21:25:59.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.408232 28672 fs_report.cc:389] FS layout report
21:25:59.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:25:59.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-1/wal
21:25:59.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-1/wal
21:25:59.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-1/data/data
21:25:59.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:25:59.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:25:59.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:25:59.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:25:59.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:25:59.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:25:59.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:25:59.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:25:59.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:25:59.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:25:59.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:25:59.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.459264 28672 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:25:59.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.460891 28672 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:25:59.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.461519 28672 kserver.cc:163] Server-wide thread pool size limit: 3276
21:25:59.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.464090 28672 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:25:59.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.468497 28672 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:25:59.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.468731 28672 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:25:59.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.469050 28672 ts_tablet_manager.cc:616] Registered 0 tablets
21:25:59.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.469238 28672 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:25:59.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.630235 28672 rpc_server.cc:307] RPC server started. Bound to: 127.27.145.66:45447
21:25:59.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.630326 28801 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.145.66:45447 every 8 connection(s)
21:25:59.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.632788 28672 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-1/data/info.pb
21:25:59.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.635179 28229 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 28672
21:25:59.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.635805 28229 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-1/wal/instance
21:25:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.648604 28229 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:25:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:25:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-2/wal
21:25:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-2/data
21:25:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:25:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:25:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:25:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:25:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-2/logs
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-2/data/info.pb
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.145.67:0
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.145.67
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.145.67
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.145.126:42349,127.27.145.125:41401,127.27.145.124:44363
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.145.84:33349
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:25:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:25:59.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.676759 28803 heartbeater.cc:344] Connected to a master server at 127.27.145.125:41401
21:25:59.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.677294 28803 heartbeater.cc:461] Registering TS with master...
21:25:59.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.678103 28802 heartbeater.cc:344] Connected to a master server at 127.27.145.124:44363
21:25:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.679869 28802 heartbeater.cc:461] Registering TS with master...
21:25:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.680020 28803 heartbeater.cc:507] Master 127.27.145.125:41401 requested a full tablet report, sending...
21:25:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.680899 28802 heartbeater.cc:507] Master 127.27.145.124:44363 requested a full tablet report, sending...
21:25:59.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.683799 28429 ts_manager.cc:194] Registered new tserver with Master: 50e9ad1f1f4a4ed0b4c8a8bea5224b20 (127.27.145.66:45447)
21:25:59.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.683781 28349 ts_manager.cc:194] Registered new tserver with Master: 50e9ad1f1f4a4ed0b4c8a8bea5224b20 (127.27.145.66:45447)
21:25:59.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.685292 28804 heartbeater.cc:344] Connected to a master server at 127.27.145.126:42349
21:25:59.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.685626 28804 heartbeater.cc:461] Registering TS with master...
21:25:59.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.686159 28429 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.27.145.66:36309
21:25:59.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.686575 28804 heartbeater.cc:507] Master 127.27.145.126:42349 requested a full tablet report, sending...
21:25:59.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:25:59.689271 28274 ts_manager.cc:194] Registered new tserver with Master: 50e9ad1f1f4a4ed0b4c8a8bea5224b20 (127.27.145.66:45447)
21:26:00.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:00.006472 28812 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:26:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:00.006946 28812 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:26:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:00.007375 28812 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:26:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:00.037930 28812 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:26:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:00.038691 28812 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.145.67
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:00.073688 28812 tablet_server_runner.cc:78] Tablet server non-default flags:
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.145.84:33349
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-2/data
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-2/wal
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.145.67:0
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-2/data/info.pb
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.145.67
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.145.126:42349,127.27.145.125:41401,127.27.145.124:44363
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.145.67
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-2/logs
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:26:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:26:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:26:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:26:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:26:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:26:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:26:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:26:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:26:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:00.074987 28812 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:26:00.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:00.076452 28812 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:26:00.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:00.090999 28818 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:00.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:00.690933 28802 heartbeater.cc:499] Master 127.27.145.124:44363 was elected leader, sending a full tablet report...
21:26:01.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:00.091427 28819 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:01.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:01.405292 28821 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:01.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:01.410579 28812 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.320s user 0.412s sys 0.895s
21:26:01.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:01.410990 28812 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.321s user 0.412s sys 0.895s
21:26:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:01.411760 28820 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1318 milliseconds
21:26:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.411772 28812 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:26:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.413496 28812 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:26:01.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.418965 28812 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:26:01.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.420336 28812 hybrid_clock.cc:648] HybridClock initialized: now 1770067561420291 us; error 59 us; skew 500 ppm
21:26:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.421348 28812 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:26:01.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.428577 28812 webserver.cc:492] Webserver started at http://127.27.145.67:44409/ using document root <none> and password file <none>
21:26:01.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.429541 28812 fs_manager.cc:362] Metadata directory not provided
21:26:01.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.430584 28812 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:26:01.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.431162 28812 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:26:01.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.436918 28812 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-2/data/instance:
21:26:01.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5420bf0b9ca34b3ebe5f43a7f9d9d2e4"
21:26:01.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:01 on dist-test-slave-3trw"
21:26:01.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.438336 28812 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-2/wal/instance:
21:26:01.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5420bf0b9ca34b3ebe5f43a7f9d9d2e4"
21:26:01.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:01 on dist-test-slave-3trw"
21:26:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.446291 28812 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.002s
21:26:01.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.452198 28829 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:26:01.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.453342 28812 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
21:26:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.453680 28812 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-2/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-2/wal
21:26:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5420bf0b9ca34b3ebe5f43a7f9d9d2e4"
21:26:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:01 on dist-test-slave-3trw"
21:26:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.454093 28812 fs_report.cc:389] FS layout report
21:26:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:26:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-2/wal
21:26:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-2/wal
21:26:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-2/data/data
21:26:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:26:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:26:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:26:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:26:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:26:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:26:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:26:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:26:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:26:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:26:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:26:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.523097 28812 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:26:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.524541 28812 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:26:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.524940 28812 kserver.cc:163] Server-wide thread pool size limit: 3276
21:26:01.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.527433 28812 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:26:01.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.531512 28812 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:26:01.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.531709 28812 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:26:01.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.531900 28812 ts_tablet_manager.cc:616] Registered 0 tablets
21:26:01.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.532034 28812 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:26:01.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.695676 28812 rpc_server.cc:307] RPC server started. Bound to: 127.27.145.67:44901
21:26:01.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.695766 28941 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.145.67:44901 every 8 connection(s)
21:26:01.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.698388 28812 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-2/data/info.pb
21:26:01.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.708482 28229 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 28812
21:26:01.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.709045 28229 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster8707490685475150806/ts-2/wal/instance
21:26:01.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.763550 28942 heartbeater.cc:344] Connected to a master server at 127.27.145.124:44363
21:26:01.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.763855 28945 heartbeater.cc:344] Connected to a master server at 127.27.145.126:42349
21:26:01.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.764166 28942 heartbeater.cc:461] Registering TS with master...
21:26:01.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.764807 28945 heartbeater.cc:461] Registering TS with master...
21:26:01.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.765605 28942 heartbeater.cc:507] Master 127.27.145.124:44363 requested a full tablet report, sending...
21:26:01.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.765746 28945 heartbeater.cc:507] Master 127.27.145.126:42349 requested a full tablet report, sending...
21:26:01.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.768957 28274 ts_manager.cc:194] Registered new tserver with Master: 5420bf0b9ca34b3ebe5f43a7f9d9d2e4 (127.27.145.67:44901)
21:26:01.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.769583 28943 heartbeater.cc:344] Connected to a master server at 127.27.145.125:41401
21:26:01.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.769941 28943 heartbeater.cc:461] Registering TS with master...
21:26:01.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.770560 28429 ts_manager.cc:194] Registered new tserver with Master: 5420bf0b9ca34b3ebe5f43a7f9d9d2e4 (127.27.145.67:44901)
21:26:01.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.771339 28943 heartbeater.cc:507] Master 127.27.145.125:41401 requested a full tablet report, sending...
21:26:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.772818 28429 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.27.145.67:45431
21:26:01.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.773558 28349 ts_manager.cc:194] Registered new tserver with Master: 5420bf0b9ca34b3ebe5f43a7f9d9d2e4 (127.27.145.67:44901)
21:26:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:01.781750 28229 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
21:26:01.794 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:26:01.795 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
21:26:01.809 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.27.145.126"
port: 42349
}
bound_http_address {
host: "127.27.145.126"
port: 46873
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.27.145.125"
port: 41401
}
bound_http_address {
host: "127.27.145.125"
port: 38665
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.27.145.124"
port: 44363
}
bound_http_address {
host: "127.27.145.124"
port: 32923
}
}
}
21:26:01.814 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:26:01.817 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.27.145.65"
port: 44013
}
bound_http_address {
host: "127.27.145.65"
port: 40451
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.27.145.66"
port: 45447
}
bound_http_address {
host: "127.27.145.66"
port: 36233
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.27.145.67"
port: 44901
}
bound_http_address {
host: "127.27.145.67"
port: 44409
}
}
}
21:26:01.818 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:26:01.997 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
21:26:02.022 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
21:26:02.139 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskurmOAe/test-tmp
21:26:02.140 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:26:02.140 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskurmOAe/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:26:02.143 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
21:26:02.481 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:02.481 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:26:02.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:02.484375 28953 mini_oidc.cc:150] Starting JWKS server
21:26:02.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:02.488195 28953 webserver.cc:492] Webserver started at http://127.0.0.1:45551/ using document root <none> and password file <none>
21:26:02.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:02.488708 28953 mini_oidc.cc:186] Starting OIDC Discovery server
21:26:02.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:02.490907 28953 webserver.cc:492] Webserver started at http://127.0.0.1:35771/ using document root <none> and password file <none>
21:26:02.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
21:26:02.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/principal' for realm 'KRBTEST.COM',
21:26:02.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
21:26:02.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:02 dist-test-slave-3trw krb5kdc[28965](info): setting up network...
21:26:02.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
21:26:02.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:02 dist-test-slave-3trw krb5kdc[28965](info): set up 2 sockets
21:26:02.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:02 dist-test-slave-3trw krb5kdc[28965](info): commencing operation
21:26:02.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
21:26:02.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:02.776233 28942 heartbeater.cc:499] Master 127.27.145.124:44363 was elected leader, sending a full tablet report...
21:26:04.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:04.603778 28953 mini_kdc.cc:121] Time spent starting KDC: real 2.075s user 0.005s sys 0.013s
21:26:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
21:26:04.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:26:04.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
21:26:04.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
21:26:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:26:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
21:26:04.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
21:26:04.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:26:04.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
21:26:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:26:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/test-user.keytab.
21:26:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/test-user.keytab.
21:26:04.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:04 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067564, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:04.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:26:04.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-02-02T21:26:04Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:26:04.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-02-02T21:26:04Z Disabled control of system clock
21:26:04.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.70.126@KRBTEST.COM; defaulting to no policy
21:26:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.70.126@KRBTEST.COM" created.
21:26:04.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:04.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.70.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:04.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.70.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.70.126@KRBTEST.COM; defaulting to no policy
21:26:04.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:04.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.70.126@KRBTEST.COM" created.
21:26:04.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:04.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.70.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:04.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.70.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:04.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:04.834522 28953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:04.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/logs
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data/info.pb
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.126:45535
21:26:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.126
21:26:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:26:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.126
21:26:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:26:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:26:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:26:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:26:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:26:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:05.147642 28982 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:26:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:05.148164 28982 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:26:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:05.148756 28982 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:26:05.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:05.180749 28982 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:26:05.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:05.180977 28982 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:26:05.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:05.181243 28982 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:26:05.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:05.181425 28982 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:26:05.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:05.181602 28982 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:26:05.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:05.181775 28982 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:26:05.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:05.181986 28982 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45551/jwks.json
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:05.218742 28982 master_runner.cc:386] Master server non-default flags:
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.126
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.126:45535
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data/info.pb
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.126
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/logs
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:26:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:26:05.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:26:05.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:26:05.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:05.219894 28982 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:26:05.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:05.220834 28982 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:26:05.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:05.232069 28988 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:06.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:05.240165 28991 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:06.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:05.233114 28989 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:06.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:06.520068 28990 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1282 milliseconds
21:26:06.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.520175 28982 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:26:06.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.521457 28982 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:26:06.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.523839 28982 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:26:06.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.525238 28982 hybrid_clock.cc:648] HybridClock initialized: now 1770067566525191 us; error 64 us; skew 500 ppm
21:26:06.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.526079 28982 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:26:06.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:06 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067566, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:06.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.532752 28982 init.cc:377] Logged in from keytab as kudu/127.28.70.126@KRBTEST.COM (short username kudu)
21:26:06.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.538316 28982 webserver.cc:492] Webserver started at http://127.28.70.126:46269/ using document root <none> and password file <none>
21:26:06.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.539175 28982 fs_manager.cc:362] Metadata directory not provided
21:26:06.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.539355 28982 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:26:06.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.539839 28982 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:26:06.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.544094 28982 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data/instance:
21:26:06.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "060798801f6146ffa33df46797fa9503"
21:26:06.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:06 on dist-test-slave-3trw"
21:26:06.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.545109 28982 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal/instance:
21:26:06.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "060798801f6146ffa33df46797fa9503"
21:26:06.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:06 on dist-test-slave-3trw"
21:26:06.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.553036 28982 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
21:26:06.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.558378 28999 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:26:06.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.559391 28982 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
21:26:06.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.559651 28982 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:06.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "060798801f6146ffa33df46797fa9503"
21:26:06.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:06 on dist-test-slave-3trw"
21:26:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.559916 28982 fs_report.cc:389] FS layout report
21:26:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:26:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data/data
21:26:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:26:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:26:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:26:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:26:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:26:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:26:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:26:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:26:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:26:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:26:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:26:06.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.630774 28982 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:26:06.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.637519 28982 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:26:06.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.637993 28982 kserver.cc:163] Server-wide thread pool size limit: 3276
21:26:06.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.724278 28982 rpc_server.cc:307] RPC server started. Bound to: 127.28.70.126:45535
21:26:06.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.724327 29050 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.70.126:45535 every 8 connection(s)
21:26:06.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.727314 28982 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data/info.pb
21:26:06.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.730732 28953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 28982
21:26:06.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.731294 28953 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal/instance
21:26:06.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.738662 29051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:26:06.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.70.125@KRBTEST.COM; defaulting to no policy
21:26:06.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:06.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.70.125@KRBTEST.COM" created.
21:26:06.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.756247 29051 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } has no permanent_uuid. Determining permanent_uuid...
21:26:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.70.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.70.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:06.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:06 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067566, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.126@KRBTEST.COM for kudu/127.28.70.126@KRBTEST.COM
21:26:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.70.125@KRBTEST.COM; defaulting to no policy
21:26:06.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:06.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.70.125@KRBTEST.COM" created.
21:26:06.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.799150 29051 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } has no permanent_uuid. Determining permanent_uuid...
21:26:06.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:06.802067 29001 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.70.125:34925: connect: Connection refused (error 111)
21:26:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:06.813774 29051 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.125:34925: Network error: Client connection negotiation failed: client connection to 127.28.70.125:34925: connect: Connection refused (error 111)
21:26:06.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:06.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.70.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:06.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.70.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:06.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.832015 28953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:06.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:06.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:06.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data
21:26:06.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:26:06.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:26:06.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/logs
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data/info.pb
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.125:34925
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.125
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.125
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:26:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:26:06.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.863418 29051 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } attempt: 1
21:26:06.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:06.867628 29051 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.125:34925: Network error: Client connection negotiation failed: client connection to 127.28.70.125:34925: connect: Connection refused (error 111)
21:26:06.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:06.936510 29051 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } attempt: 2
21:26:06.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:06.941008 29051 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.125:34925: Network error: Client connection negotiation failed: client connection to 127.28.70.125:34925: connect: Connection refused (error 111)
21:26:07.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:07.032400 29051 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } attempt: 3
21:26:07.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:07.036424 29051 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.125:34925: Network error: Client connection negotiation failed: client connection to 127.28.70.125:34925: connect: Connection refused (error 111)
21:26:07.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:07.158552 29060 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:26:07.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:07.159065 29060 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:26:07.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:07.159675 29060 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:26:07.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:07.179828 29051 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } attempt: 4
21:26:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:07.184477 29051 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.125:34925: Network error: Client connection negotiation failed: client connection to 127.28.70.125:34925: connect: Connection refused (error 111)
21:26:07.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:07.195223 29060 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:26:07.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:07.195454 29060 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:26:07.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:07.195780 29060 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:26:07.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:07.195993 29060 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:26:07.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:07.196189 29060 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:26:07.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:07.196388 29060 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:26:07.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:07.196535 29060 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45551/jwks.json
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:07.233982 29060 master_runner.cc:386] Master server non-default flags:
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.125
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.125:34925
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:26:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:26:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:26:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data/info.pb
21:26:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.125
21:26:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:26:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:26:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:26:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:26:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/logs
21:26:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:26:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:26:07.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:26:07.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:26:07.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:26:07.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:26:07.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:26:07.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:26:07.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:26:07.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:07.235394 29060 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:26:07.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:07.236357 29060 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:26:07.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:07.248553 29067 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:07.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:07.483866 29051 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } attempt: 5
21:26:07.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:07.487800 29051 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.125:34925: Network error: Client connection negotiation failed: client connection to 127.28.70.125:34925: connect: Connection refused (error 111)
21:26:08.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.035244 29051 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } attempt: 6
21:26:08.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:08.054296 29051 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.125:34925: Network error: Client connection negotiation failed: client connection to 127.28.70.125:34925: connect: Connection refused (error 111)
21:26:08.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:07.250782 29068 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:08.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:08.595168 29070 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:08.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:08.598399 29069 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1348 milliseconds
21:26:08.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.598487 29060 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:26:08.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.599815 29060 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:26:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.602885 29060 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:26:08.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.604300 29060 hybrid_clock.cc:648] HybridClock initialized: now 1770067568604266 us; error 36 us; skew 500 ppm
21:26:08.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.605147 29060 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:26:08.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:08 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067568, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:08.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.611395 29060 init.cc:377] Logged in from keytab as kudu/127.28.70.125@KRBTEST.COM (short username kudu)
21:26:08.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.617442 29060 webserver.cc:492] Webserver started at http://127.28.70.125:38211/ using document root <none> and password file <none>
21:26:08.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.618458 29060 fs_manager.cc:362] Metadata directory not provided
21:26:08.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.618692 29060 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:26:08.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.619180 29060 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:26:08.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.623507 29060 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data/instance:
21:26:08.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c07e7962716a40a8bb9a84004a578cfb"
21:26:08.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:08 on dist-test-slave-3trw"
21:26:08.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.624639 29060 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal/instance:
21:26:08.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c07e7962716a40a8bb9a84004a578cfb"
21:26:08.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:08 on dist-test-slave-3trw"
21:26:08.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.632812 29060 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
21:26:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.639066 29082 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:26:08.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.640321 29060 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
21:26:08.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.640604 29060 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:08.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c07e7962716a40a8bb9a84004a578cfb"
21:26:08.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:08 on dist-test-slave-3trw"
21:26:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.640877 29060 fs_report.cc:389] FS layout report
21:26:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:26:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data/data
21:26:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:26:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:26:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:26:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:26:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:26:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:26:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:26:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:26:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:26:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:26:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:26:08.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.699450 29060 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:26:08.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.709728 29060 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:26:08.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.710665 29060 kserver.cc:163] Server-wide thread pool size limit: 3276
21:26:08.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.804859 29133 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.70.125:34925 every 8 connection(s)
21:26:08.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.804701 29060 rpc_server.cc:307] RPC server started. Bound to: 127.28.70.125:34925
21:26:08.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.810845 29060 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data/info.pb
21:26:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.813812 28953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 29060
21:26:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.814303 28953 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal/instance
21:26:08.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.819522 29134 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:26:08.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.70.124@KRBTEST.COM; defaulting to no policy
21:26:08.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:08.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.70.124@KRBTEST.COM" created.
21:26:08.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.836464 29134 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } has no permanent_uuid. Determining permanent_uuid...
21:26:08.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:08.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.70.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:08.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.70.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:08.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:08 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067568, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.125@KRBTEST.COM for kudu/127.28.70.126@KRBTEST.COM
21:26:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.70.124@KRBTEST.COM; defaulting to no policy
21:26:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.70.124@KRBTEST.COM" created.
21:26:08.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.888433 29134 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } has no permanent_uuid. Determining permanent_uuid...
21:26:08.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:08 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067568, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.125@KRBTEST.COM for kudu/127.28.70.125@KRBTEST.COM
21:26:08.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:08.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.70.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:08.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.70.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.907737 28953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/logs
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data/info.pb
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.124:42699
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.124
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.124
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:26:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:26:08.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.916637 29134 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } has no permanent_uuid. Determining permanent_uuid...
21:26:08.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:08.918920 29085 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:08.923729 29134 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:08.973155 29134 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } attempt: 1
21:26:08.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:08.977033 29134 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:09.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:09.045476 29134 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } attempt: 2
21:26:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.049322 29134 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:09.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:09.114861 29051 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } attempt: 7
21:26:09.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:09 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067566, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.126@KRBTEST.COM for kudu/127.28.70.125@KRBTEST.COM
21:26:09.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:09.140751 29134 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } attempt: 3
21:26:09.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.148880 29134 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:09.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:09.151302 29051 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } has no permanent_uuid. Determining permanent_uuid...
21:26:09.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.162314 29051 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:09.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:09.220913 29051 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } attempt: 1
21:26:09.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.227491 29051 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:09.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.264338 29143 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:26:09.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.265285 29143 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:26:09.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.266127 29143 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:26:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:09.292328 29134 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } attempt: 4
21:26:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.298202 29134 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.300943 29143 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:26:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.301160 29143 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:26:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.301430 29143 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:26:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.301623 29143 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:26:09.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.301815 29143 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:26:09.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.302088 29143 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:26:09.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.302244 29143 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45551/jwks.json
21:26:09.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:09.308917 29051 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } attempt: 2
21:26:09.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.314401 29051 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:09.349810 29143 master_runner.cc:386] Master server non-default flags:
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.124
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.124:42699
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data/info.pb
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.124
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:26:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:26:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:26:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:26:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/logs
21:26:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:26:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:26:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:26:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:26:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:26:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:26:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:26:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:26:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:26:09.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:09.351189 29143 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:26:09.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:09.352263 29143 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:26:09.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.372465 29153 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:09.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:09.399852 29051 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } attempt: 3
21:26:09.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.407452 29051 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:09.548096 29051 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } attempt: 4
21:26:09.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.559633 29051 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:09.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:09.597685 29134 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } attempt: 5
21:26:09.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.608348 29134 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:09.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:09.843327 29051 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } attempt: 5
21:26:09.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.852461 29051 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:10.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:10.156080 29134 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } attempt: 6
21:26:10.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:10.184969 29134 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:10.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:10.405097 29051 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } attempt: 6
21:26:10.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:10.436906 29051 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:11.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:10.774654 29152 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 29143
21:26:11.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:11.215487 29152 kernel_stack_watchdog.cc:198] Thread 29143 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 393ms:
21:26:11.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:26:11.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:26:11.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:26:11.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:26:11.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:11.216432 29143 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.838s user 0.665s sys 1.075s
21:26:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:11.216902 29143 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.839s user 0.665s sys 1.075s
21:26:11.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:09.377450 29154 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:11.219388 29156 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:11.222970 29155 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1838 milliseconds
21:26:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.222995 29143 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:26:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.224476 29143 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:26:11.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.227464 29143 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:26:11.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.228796 29143 hybrid_clock.cc:648] HybridClock initialized: now 1770067571228744 us; error 47 us; skew 500 ppm
21:26:11.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.229601 29143 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:26:11.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:11 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067571, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:11.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.235263 29143 init.cc:377] Logged in from keytab as kudu/127.28.70.124@KRBTEST.COM (short username kudu)
21:26:11.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.241111 29143 webserver.cc:492] Webserver started at http://127.28.70.124:34373/ using document root <none> and password file <none>
21:26:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.242096 29143 fs_manager.cc:362] Metadata directory not provided
21:26:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.242367 29143 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:26:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.242798 29143 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:26:11.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.245596 29134 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } attempt: 7
21:26:11.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.247331 29143 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data/instance:
21:26:11.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0da549b46dae45938c88c7a89dae6e27"
21:26:11.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:11 on dist-test-slave-3trw"
21:26:11.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.248317 29143 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal/instance:
21:26:11.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0da549b46dae45938c88c7a89dae6e27"
21:26:11.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:11 on dist-test-slave-3trw"
21:26:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:11.252681 29134 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:11.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.256287 29143 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.005s
21:26:11.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.261783 29173 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:26:11.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.262924 29143 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
21:26:11.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.263232 29143 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:11.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0da549b46dae45938c88c7a89dae6e27"
21:26:11.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:11 on dist-test-slave-3trw"
21:26:11.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.263522 29143 fs_report.cc:389] FS layout report
21:26:11.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:26:11.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:11.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data/data
21:26:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:26:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:26:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:26:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:26:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:26:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:26:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:26:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:26:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:26:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:26:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:26:11.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.322373 29143 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:26:11.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.328141 29143 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:26:11.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.328536 29143 kserver.cc:163] Server-wide thread pool size limit: 3276
21:26:11.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.414512 29143 rpc_server.cc:307] RPC server started. Bound to: 127.28.70.124:42699
21:26:11.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.414547 29224 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.70.124:42699 every 8 connection(s)
21:26:11.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.417093 29143 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data/info.pb
21:26:11.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.423988 29225 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:26:11.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.427414 28953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 29143
21:26:11.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.427769 28953 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal/instance
21:26:11.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.437110 29225 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } has no permanent_uuid. Determining permanent_uuid...
21:26:11.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:11 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067564, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.70.126@KRBTEST.COM
21:26:11.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.470479 29051 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } attempt: 7
21:26:11.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:11 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067571, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.124@KRBTEST.COM for kudu/127.28.70.126@KRBTEST.COM
21:26:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:11 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067566, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.126@KRBTEST.COM for kudu/127.28.70.124@KRBTEST.COM
21:26:11.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:11 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067564, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.70.125@KRBTEST.COM
21:26:11.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.497593 29225 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } has no permanent_uuid. Determining permanent_uuid...
21:26:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:11 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067571, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.124@KRBTEST.COM for kudu/127.28.70.125@KRBTEST.COM
21:26:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.520213 29051 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Bootstrap starting.
21:26:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:11 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067564, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.70.124@KRBTEST.COM
21:26:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.528116 29051 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Neither blocks nor log segments found. Creating new log.
21:26:11.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.530823 29051 log.cc:826] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Log is configured to *not* fsync() on all Append() calls
21:26:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.532279 29225 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } has no permanent_uuid. Determining permanent_uuid...
21:26:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.537439 29051 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: No bootstrap required, opened a new log
21:26:11.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:11 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067571, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.124@KRBTEST.COM for kudu/127.28.70.124@KRBTEST.COM
21:26:11.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.564258 29051 raft_consensus.cc:359] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:11.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.564934 29051 raft_consensus.cc:385] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:26:11.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.565270 29051 raft_consensus.cc:740] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 060798801f6146ffa33df46797fa9503, State: Initialized, Role: FOLLOWER
21:26:11.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.566100 29051 consensus_queue.cc:260] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.567003 29225 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: Bootstrap starting.
21:26:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.575572 29237 sys_catalog.cc:455] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.576514 29237 sys_catalog.cc:458] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: This master's current role is: FOLLOWER
21:26:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.577292 29051 sys_catalog.cc:565] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: configured and running, proceeding with master startup.
21:26:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.578979 29225 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: Neither blocks nor log segments found. Creating new log.
21:26:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.583096 29225 log.cc:826] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: Log is configured to *not* fsync() on all Append() calls
21:26:11.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.593698 29225 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: No bootstrap required, opened a new log
21:26:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:11.617547 29249 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:26:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:11.617955 29249 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:26:11.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.627219 29225 raft_consensus.cc:359] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.630000 29225 raft_consensus.cc:385] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:26:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.630515 29225 raft_consensus.cc:740] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0da549b46dae45938c88c7a89dae6e27, State: Initialized, Role: FOLLOWER
21:26:11.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.631623 29225 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:11.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.647516 29251 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:11.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.648391 29251 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: This master's current role is: FOLLOWER
21:26:11.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.614888 28982 master_runner.cc:186] Error getting master registration for 127.28.70.125:34925: OK, instance_id { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" instance_seqno: 1770067568777641 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:26:11.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:11.648906 29005 tablet.cc:2378] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:26:11.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.649225 28982 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:26:11.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.650043 29225 sys_catalog.cc:565] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: configured and running, proceeding with master startup.
21:26:11.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.679641 29237 raft_consensus.cc:493] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:26:11.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.680326 29237 raft_consensus.cc:515] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:11.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.684276 29237 leader_election.cc:290] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c07e7962716a40a8bb9a84004a578cfb (127.28.70.125:34925), 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699)
21:26:11.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.686163 29200 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "060798801f6146ffa33df46797fa9503" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0da549b46dae45938c88c7a89dae6e27" is_pre_election: true
21:26:11.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.687040 29200 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 060798801f6146ffa33df46797fa9503 in term 0.
21:26:11.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.688532 29002 leader_election.cc:304] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 060798801f6146ffa33df46797fa9503, 0da549b46dae45938c88c7a89dae6e27; no voters:
21:26:11.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.689874 29237 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:26:11.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.690393 29237 raft_consensus.cc:493] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:26:11.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.692613 29237 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 0 FOLLOWER]: Advancing to term 1
21:26:11.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.702941 29237 raft_consensus.cc:515] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:11.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:11.721529 29262 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:26:11.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:11.721906 29262 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:26:11.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.721771 29237 leader_election.cc:290] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 1 election: Requested vote from peers c07e7962716a40a8bb9a84004a578cfb (127.28.70.125:34925), 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699)
21:26:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.723366 29200 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "060798801f6146ffa33df46797fa9503" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0da549b46dae45938c88c7a89dae6e27"
21:26:11.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.723897 29200 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [term 0 FOLLOWER]: Advancing to term 1
21:26:11.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.731629 29200 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 060798801f6146ffa33df46797fa9503 in term 1.
21:26:11.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.732802 29002 leader_election.cc:304] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 060798801f6146ffa33df46797fa9503, 0da549b46dae45938c88c7a89dae6e27; no voters:
21:26:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.740329 29237 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 1 FOLLOWER]: Leader election won for term 1
21:26:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.781493 29237 raft_consensus.cc:697] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 1 LEADER]: Becoming Leader. State: Replica: 060798801f6146ffa33df46797fa9503, State: Running, Role: LEADER
21:26:11.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.783231 29237 consensus_queue.cc:237] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:11.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.722702 29143 master_runner.cc:186] Error getting master registration for 127.28.70.125:34925: OK, instance_id { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" instance_seqno: 1770067568777641 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:26:11.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.790596 29143 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:26:11.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.797693 29264 sys_catalog.cc:455] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 060798801f6146ffa33df46797fa9503. Latest consensus state: current_term: 1 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:11.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.798743 29264 sys_catalog.cc:458] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: This master's current role is: LEADER
21:26:11.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.832931 29266 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:26:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:11.834611 29179 tablet.cc:2378] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:26:11.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.882778 29266 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:26:11.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.900139 29200 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [term 1 FOLLOWER]: Refusing update from remote peer 060798801f6146ffa33df46797fa9503: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:26:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.905795 29264 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:26:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.931730 29251 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 060798801f6146ffa33df46797fa9503. Latest consensus state: current_term: 1 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.932736 29251 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: This master's current role is: FOLLOWER
21:26:11.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.940656 29267 mvcc.cc:204] Tried to move back new op lower bound from 7250196774483931136 to 7250196774035202048. Current Snapshot: MvccSnapshot[applied={T|T < 7250196774483931136}]
21:26:11.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.947822 29251 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:11.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.948513 29251 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: This master's current role is: FOLLOWER
21:26:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.953490 29268 mvcc.cc:204] Tried to move back new op lower bound from 7250196774483931136 to 7250196774035202048. Current Snapshot: MvccSnapshot[applied={T|T < 7250196774483931136 or (T in {7250196774483931136})}]
21:26:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.960969 29264 sys_catalog.cc:455] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.961732 29264 sys_catalog.cc:458] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: This master's current role is: LEADER
21:26:11.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.964207 29266 catalog_manager.cc:1357] Generated new cluster ID: 1d6e92849c0c462dbb1161273536d9ce
21:26:11.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:11.964493 29266 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:26:12.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:12.002017 29266 catalog_manager.cc:1380] Generated new certificate authority record
21:26:12.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:12.006323 29266 catalog_manager.cc:1514] Loading token signing keys...
21:26:12.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:12.044762 29266 catalog_manager.cc:6027] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Generated new TSK 0
21:26:12.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:12.046191 29266 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:26:12.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.70.65@KRBTEST.COM; defaulting to no policy
21:26:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:12.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.70.65@KRBTEST.COM" created.
21:26:12.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:12.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.70.65 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:12.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.70.65 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.70.65@KRBTEST.COM; defaulting to no policy
21:26:12.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:12.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.70.65@KRBTEST.COM" created.
21:26:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.70.65 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.70.65 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:12.234814 28953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-0/wal
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-0/data
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-0/logs
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-0/data/info.pb
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:12.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:12.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:12.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:26:12.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:26:12.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.65:0
21:26:12.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.70.65
21:26:12.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.65
21:26:12.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:26:12.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:12.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:12.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.65
21:26:12.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:12.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:26:12.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:26:12.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:26:12.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:12.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:12.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:12.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:12.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:26:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:12.731602 29262 catalog_manager.cc:1269] Loaded cluster ID: 1d6e92849c0c462dbb1161273536d9ce
21:26:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:12.732335 29262 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: loading cluster ID for follower catalog manager: success
21:26:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:12.756513 29262 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: acquiring CA information for follower catalog manager: success
21:26:12.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:12.775709 29262 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:26:12.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:12.861809 29280 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:26:12.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:12.862495 29280 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:26:12.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:12.863539 29280 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:26:12.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:12.931591 29280 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:26:12.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:12.933789 29280 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.70.65
21:26:12.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:12.999441 29280 tablet_server_runner.cc:78] Tablet server non-default flags:
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-0/data
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-0/wal
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.65
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.65:0
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-0/data/info.pb
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.65
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.70.65
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-0/logs
21:26:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:13.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:26:13.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:26:13.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:26:13.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:26:13.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:26:13.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:26:13.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:26:13.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:26:13.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:26:13.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.001138 29280 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:26:13.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.002512 29280 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:26:13.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:13.034732 29288 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:13.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:13.309046 29001 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.28.70.125:34925 timed out after 1.588s (SENT) [suppressed 13 similar messages]
21:26:13.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.343151 29134 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } attempt: 8
21:26:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:13 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067568, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.125@KRBTEST.COM for kudu/127.28.70.124@KRBTEST.COM
21:26:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.420497 29134 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: Bootstrap starting.
21:26:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.428241 29134 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: Neither blocks nor log segments found. Creating new log.
21:26:13.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.431331 29134 log.cc:826] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: Log is configured to *not* fsync() on all Append() calls
21:26:13.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.455849 29134 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: No bootstrap required, opened a new log
21:26:13.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.520951 29134 raft_consensus.cc:359] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:13.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.522089 29134 raft_consensus.cc:385] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:26:13.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.524593 29134 raft_consensus.cc:740] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c07e7962716a40a8bb9a84004a578cfb, State: Initialized, Role: FOLLOWER
21:26:13.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.525717 29134 consensus_queue.cc:260] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:13.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.541040 29298 sys_catalog.cc:455] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.543100 29134 sys_catalog.cc:565] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: configured and running, proceeding with master startup.
21:26:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.542078 29298 sys_catalog.cc:458] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: This master's current role is: FOLLOWER
21:26:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.543397 29108 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "060798801f6146ffa33df46797fa9503" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c07e7962716a40a8bb9a84004a578cfb"
21:26:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.544346 29108 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 0 FOLLOWER]: Advancing to term 1
21:26:13.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.546552 29106 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "060798801f6146ffa33df46797fa9503" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c07e7962716a40a8bb9a84004a578cfb"
21:26:13.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.548213 29109 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "060798801f6146ffa33df46797fa9503" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c07e7962716a40a8bb9a84004a578cfb" is_pre_election: true
21:26:13.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:13.549335 29109 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:33176 (request call id 2) took 1865 ms (client timeout 1991 ms). Trace:
21:26:13.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:13.549804 29109 rpcz_store.cc:269] 0202 21:26:11.684258 (+ 0us) service_pool.cc:168] Inserting onto call queue
21:26:13.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 21:26:11.684405 (+ 147us) service_pool.cc:225] Handling call
21:26:13.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 21:26:13.549253 (+1864848us) inbound_call.cc:173] Queueing success response
21:26:13.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
21:26:13.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.567870 29108 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 060798801f6146ffa33df46797fa9503 in term 1.
21:26:13.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.576006 29107 raft_consensus.cc:1275] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 1 FOLLOWER]: Refusing update from remote peer 060798801f6146ffa33df46797fa9503: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:26:13.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.583799 29107 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:33176 (request call id 4) took 1684 ms. Trace:
21:26:13.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.584623 29107 rpcz_store.cc:276] 0202 21:26:11.899693 (+ 0us) service_pool.cc:168] Inserting onto call queue
21:26:13.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 21:26:11.899861 (+ 168us) service_pool.cc:225] Handling call
21:26:13.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 21:26:13.575119 (+1675258us) raft_consensus.cc:1471] Updating replica for 0 ops
21:26:13.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 21:26:13.576491 (+ 1372us) raft_consensus.cc:1697] Filling consensus response to leader.
21:26:13.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 21:26:13.583420 (+ 6929us) spinlock_profiling.cc:243] Waited 28.7 ms on lock 0x7b5400030550. stack: 00007fb301334373 00007fb301334270 00007fb300ea9574 0000000000508744 00000000005086d9 00007fb3083855a1 00007fb30841b52d 00007fb30944af49 00007fb307affd34 00007fb307affcc2 00007fb307affbf1 00007fb307affb6c 00007fb307afee22 00007fb30611a34c 00007fb306119786 00007fb30611912f
21:26:13.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 21:26:13.583715 (+ 295us) inbound_call.cc:173] Queueing success response
21:26:13.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"spinlock_wait_cycles":65962880}
21:26:13.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:13.587042 29108 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:33176 (request call id 3) took 1864 ms (client timeout 1588 ms). Trace:
21:26:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:13.594312 29108 rpcz_store.cc:269] 0202 21:26:11.723086 (+ 0us) service_pool.cc:168] Inserting onto call queue
21:26:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 21:26:11.723243 (+ 157us) service_pool.cc:225] Handling call
21:26:13.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 21:26:13.586948 (+1863705us) inbound_call.cc:173] Queueing success response
21:26:13.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
21:26:13.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.612651 29301 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:26:13.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:13.652343 29002 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.70.124:42699, user_credentials={real_user=slave}} blocked reactor thread for 45223.4us
21:26:13.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.748488 29298 sys_catalog.cc:455] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: SysCatalogTable state changed. Reason: New leader 060798801f6146ffa33df46797fa9503. Latest consensus state: current_term: 1 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.757601 29298 sys_catalog.cc:458] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: This master's current role is: FOLLOWER
21:26:13.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.763103 29314 sys_catalog.cc:455] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:13.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.764498 29314 sys_catalog.cc:458] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: This master's current role is: FOLLOWER
21:26:13.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:13.799693 29317 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:26:14.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:14.232863 29318 sys_catalog.cc:455] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:14.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:14.233896 29318 sys_catalog.cc:458] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: This master's current role is: LEADER
21:26:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:14.837323 29317 catalog_manager.cc:1269] Loaded cluster ID: 1d6e92849c0c462dbb1161273536d9ce
21:26:14.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:14.837599 29317 catalog_manager.cc:1562] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: loading cluster ID for follower catalog manager: success
21:26:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:14.858649 29317 catalog_manager.cc:1584] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: acquiring CA information for follower catalog manager: success
21:26:14.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:14.865067 29317 catalog_manager.cc:1612] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:26:15.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:13.043241 29289 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:15.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:15.416731 29291 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:15.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.423022 29280 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:26:15.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:15.423022 29290 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2387 milliseconds
21:26:15.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.424823 29280 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:26:15.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.435743 29280 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:26:15.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.438509 29280 hybrid_clock.cc:648] HybridClock initialized: now 1770067575438452 us; error 49 us; skew 500 ppm
21:26:15.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.439602 29280 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:26:15.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:15 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067575, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:15.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.447687 29280 init.cc:377] Logged in from keytab as kudu/127.28.70.65@KRBTEST.COM (short username kudu)
21:26:15.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.490579 29280 webserver.cc:492] Webserver started at http://127.28.70.65:41911/ using document root <none> and password file <none>
21:26:15.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.492182 29280 fs_manager.cc:362] Metadata directory not provided
21:26:15.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.493678 29280 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:26:15.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.494431 29280 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:26:15.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.501061 29280 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-0/data/instance:
21:26:15.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7de4d9a040ba4467892670bc54c36b8c"
21:26:15.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:15 on dist-test-slave-3trw"
21:26:15.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.502948 29280 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-0/wal/instance:
21:26:15.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7de4d9a040ba4467892670bc54c36b8c"
21:26:15.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:15 on dist-test-slave-3trw"
21:26:15.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.519034 29280 fs_manager.cc:696] Time spent creating directory manager: real 0.015s user 0.009s sys 0.004s
21:26:15.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.532119 29333 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:26:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.533638 29280 fs_manager.cc:730] Time spent opening block manager: real 0.010s user 0.009s sys 0.000s
21:26:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.534225 29280 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-0/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-0/wal
21:26:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7de4d9a040ba4467892670bc54c36b8c"
21:26:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:15 on dist-test-slave-3trw"
21:26:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.534787 29280 fs_report.cc:389] FS layout report
21:26:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:26:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-0/wal
21:26:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-0/wal
21:26:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-0/data/data
21:26:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:26:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:26:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:26:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:26:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:26:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:26:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:26:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:26:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:26:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:26:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:26:15.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.720981 29280 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:26:15.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.737361 29280 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:26:15.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.738258 29280 kserver.cc:163] Server-wide thread pool size limit: 3276
21:26:15.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.754258 29280 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:26:15.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.770614 29280 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:26:15.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.770907 29280 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:26:15.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.771195 29280 ts_tablet_manager.cc:616] Registered 0 tablets
21:26:15.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:15.771404 29280 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:26:16.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.224355 29280 rpc_server.cc:307] RPC server started. Bound to: 127.28.70.65:44823
21:26:16.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.224546 29447 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.70.65:44823 every 8 connection(s)
21:26:16.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.227885 29280 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-0/data/info.pb
21:26:16.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.233605 28953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 29280
21:26:16.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.234313 28953 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-0/wal/instance
21:26:16.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:16 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067575, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.65@KRBTEST.COM for kudu/127.28.70.125@KRBTEST.COM
21:26:16.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:16 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067575, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.65@KRBTEST.COM for kudu/127.28.70.124@KRBTEST.COM
21:26:16.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.70.66@KRBTEST.COM; defaulting to no policy
21:26:16.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:16 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067575, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.65@KRBTEST.COM for kudu/127.28.70.126@KRBTEST.COM
21:26:16.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:16.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.70.66@KRBTEST.COM" created.
21:26:16.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.327708 29450 heartbeater.cc:344] Connected to a master server at 127.28.70.126:45535
21:26:16.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.328202 29450 heartbeater.cc:461] Registering TS with master...
21:26:16.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.329480 29450 heartbeater.cc:507] Master 127.28.70.126:45535 requested a full tablet report, sending...
21:26:16.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.332924 29015 ts_manager.cc:194] Registered new tserver with Master: 7de4d9a040ba4467892670bc54c36b8c (127.28.70.65:44823)
21:26:16.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.334878 29449 heartbeater.cc:344] Connected to a master server at 127.28.70.125:34925
21:26:16.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.335630 29449 heartbeater.cc:461] Registering TS with master...
21:26:16.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.336411 29015 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.28.70.65@KRBTEST.COM'} at 127.28.70.65:57361
21:26:16.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.336601 29449 heartbeater.cc:507] Master 127.28.70.125:34925 requested a full tablet report, sending...
21:26:16.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.338383 29448 heartbeater.cc:344] Connected to a master server at 127.28.70.124:42699
21:26:16.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.338744 29448 heartbeater.cc:461] Registering TS with master...
21:26:16.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.339738 29448 heartbeater.cc:507] Master 127.28.70.124:42699 requested a full tablet report, sending...
21:26:16.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.340925 29099 ts_manager.cc:194] Registered new tserver with Master: 7de4d9a040ba4467892670bc54c36b8c (127.28.70.65:44823)
21:26:16.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.343672 29190 ts_manager.cc:194] Registered new tserver with Master: 7de4d9a040ba4467892670bc54c36b8c (127.28.70.65:44823)
21:26:16.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:16.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.70.66 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:16.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.70.66 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:16.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:16.365046 28953 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.28.70.66: real 0.113s user 0.003s sys 0.011s
21:26:16.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.70.66@KRBTEST.COM; defaulting to no policy
21:26:16.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:16.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.70.66@KRBTEST.COM" created.
21:26:16.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:16.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.70.66 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:16.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.70.66 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.408766 28953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-1/wal
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-1/data
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-1/logs
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-1/data/info.pb
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:26:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.66:0
21:26:16.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.70.66
21:26:16.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.66
21:26:16.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:26:16.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:16.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:16.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.66
21:26:16.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:16.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:26:16.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:26:16.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:26:16.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:16.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:16.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:16.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:16.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:26:16.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:16 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067576, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:16.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.537812 28995 init.cc:295] Successfully reacquired a new kerberos TGT
21:26:16.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:16.785012 29462 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:26:16.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:16.785562 29462 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:26:16.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:16.786316 29462 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:26:16.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:16.848227 29462 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:26:16.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:16.850313 29462 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.70.66
21:26:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.903062 29462 tablet_server_runner.cc:78] Tablet server non-default flags:
21:26:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-1/data
21:26:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-1/wal
21:26:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.66
21:26:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.66:0
21:26:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:26:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:26:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:26:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-1/data/info.pb
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.66
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.70.66
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-1/logs
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:26:16.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.904675 29462 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:26:16.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:16.906040 29462 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:26:16.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:16.925031 29469 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:17.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:17.343434 29450 heartbeater.cc:499] Master 127.28.70.126:45535 was elected leader, sending a full tablet report...
21:26:18.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:18.322435 29467 debug-util.cc:398] Leaking SignalData structure 0x7b08000282a0 after lost signal to thread 29462
21:26:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:18.588999 29467 kernel_stack_watchdog.cc:198] Thread 29462 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 393ms:
21:26:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:26:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:26:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:26:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:26:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:16.930496 29468 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:18.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:18.590478 29462 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.663s user 0.423s sys 0.792s
21:26:18.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:18.591007 29462 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.664s user 0.424s sys 0.792s
21:26:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:18.594935 29471 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1667 milliseconds
21:26:18.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:18.596410 29472 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:18.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.596498 29462 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:26:18.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.597915 29462 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:26:18.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.600359 29462 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:26:18.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.601783 29462 hybrid_clock.cc:648] HybridClock initialized: now 1770067578601741 us; error 35 us; skew 500 ppm
21:26:18.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.602833 29462 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:26:18.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:18 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067578, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.610214 29462 init.cc:377] Logged in from keytab as kudu/127.28.70.66@KRBTEST.COM (short username kudu)
21:26:18.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:18 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067578, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:18.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.616039 29462 webserver.cc:492] Webserver started at http://127.28.70.66:43623/ using document root <none> and password file <none>
21:26:18.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.616077 29078 init.cc:295] Successfully reacquired a new kerberos TGT
21:26:18.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.617213 29462 fs_manager.cc:362] Metadata directory not provided
21:26:18.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.617466 29462 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:26:18.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.618002 29462 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:26:18.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.623908 29462 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-1/data/instance:
21:26:18.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "56b758d0c850420b8576cdd41f6d9658"
21:26:18.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:18 on dist-test-slave-3trw"
21:26:18.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.625329 29462 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-1/wal/instance:
21:26:18.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "56b758d0c850420b8576cdd41f6d9658"
21:26:18.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:18 on dist-test-slave-3trw"
21:26:18.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.635017 29462 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.006s sys 0.002s
21:26:18.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.642223 29484 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:26:18.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.643818 29462 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
21:26:18.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.644183 29462 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-1/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-1/wal
21:26:18.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "56b758d0c850420b8576cdd41f6d9658"
21:26:18.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:18 on dist-test-slave-3trw"
21:26:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.644551 29462 fs_report.cc:389] FS layout report
21:26:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:26:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-1/wal
21:26:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-1/wal
21:26:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-1/data/data
21:26:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:26:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:26:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:26:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:26:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:26:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:26:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:26:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:26:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:26:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:26:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:26:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.713716 29462 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:26:18.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.719367 29462 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:26:18.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.719749 29462 kserver.cc:163] Server-wide thread pool size limit: 3276
21:26:18.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.722586 29462 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:26:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.727689 29462 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:26:18.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.727922 29462 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:26:18.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.728264 29462 ts_tablet_manager.cc:616] Registered 0 tablets
21:26:18.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.728447 29462 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:26:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.977977 29462 rpc_server.cc:307] RPC server started. Bound to: 127.28.70.66:40871
21:26:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.978163 29596 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.70.66:40871 every 8 connection(s)
21:26:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.980736 29462 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-1/data/info.pb
21:26:18.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.989612 28953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 29462
21:26:18.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:18.990170 28953 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-1/wal/instance
21:26:19.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.70.67@KRBTEST.COM; defaulting to no policy
21:26:19.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:19.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.70.67@KRBTEST.COM" created.
21:26:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:19 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067578, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.66@KRBTEST.COM for kudu/127.28.70.125@KRBTEST.COM
21:26:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:19 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067578, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.66@KRBTEST.COM for kudu/127.28.70.124@KRBTEST.COM
21:26:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:19 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067578, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.66@KRBTEST.COM for kudu/127.28.70.126@KRBTEST.COM
21:26:19.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:19.051596 29598 heartbeater.cc:344] Connected to a master server at 127.28.70.125:34925
21:26:19.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:19.052124 29598 heartbeater.cc:461] Registering TS with master...
21:26:19.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:19.053272 29598 heartbeater.cc:507] Master 127.28.70.125:34925 requested a full tablet report, sending...
21:26:19.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:19.056346 29099 ts_manager.cc:194] Registered new tserver with Master: 56b758d0c850420b8576cdd41f6d9658 (127.28.70.66:40871)
21:26:19.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:19.059482 29597 heartbeater.cc:344] Connected to a master server at 127.28.70.124:42699
21:26:19.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:19.059893 29597 heartbeater.cc:461] Registering TS with master...
21:26:19.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:19.060927 29597 heartbeater.cc:507] Master 127.28.70.124:42699 requested a full tablet report, sending...
21:26:19.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:19.062568 29599 heartbeater.cc:344] Connected to a master server at 127.28.70.126:45535
21:26:19.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:19.062911 29599 heartbeater.cc:461] Registering TS with master...
21:26:19.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:19.063956 29190 ts_manager.cc:194] Registered new tserver with Master: 56b758d0c850420b8576cdd41f6d9658 (127.28.70.66:40871)
21:26:19.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:19.064255 29599 heartbeater.cc:507] Master 127.28.70.126:45535 requested a full tablet report, sending...
21:26:19.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:19.066394 29015 ts_manager.cc:194] Registered new tserver with Master: 56b758d0c850420b8576cdd41f6d9658 (127.28.70.66:40871)
21:26:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:19.067560 29015 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.28.70.66@KRBTEST.COM'} at 127.28.70.66:41885
21:26:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.70.67 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.70.67 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:19.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.70.67@KRBTEST.COM; defaulting to no policy
21:26:19.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:19.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.70.67@KRBTEST.COM" created.
21:26:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:26:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.70.67 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.70.67 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab.
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:19.107961 28953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-2/wal
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-2/data
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-2/logs
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-2/data/info.pb
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:26:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:26:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.67:0
21:26:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.70.67
21:26:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.67
21:26:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:26:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.67
21:26:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:26:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:26:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:26:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:19.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:19.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:26:19.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:19.430034 29611 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:26:19.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:19.430487 29611 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:26:19.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:19.431037 29611 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:26:19.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:19.465065 29611 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:26:19.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:19.466403 29611 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.70.67
21:26:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:19.504266 29611 tablet_server_runner.cc:78] Tablet server non-default flags:
21:26:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-2/data
21:26:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-2/wal
21:26:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.67
21:26:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.67:0
21:26:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-2/data/info.pb
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.67
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.70.67
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-2/logs
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:26:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:26:19.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:19.505975 29611 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:26:19.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:19.507119 29611 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:26:19.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:19.526041 29620 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:20.070696 29599 heartbeater.cc:499] Master 127.28.70.126:45535 was elected leader, sending a full tablet report...
21:26:21.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:21 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067581, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:21.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.241545 29167 init.cc:295] Successfully reacquired a new kerberos TGT
21:26:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:20.928292 29619 debug-util.cc:398] Leaking SignalData structure 0x7b08000282a0 after lost signal to thread 29611
21:26:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:21.290874 29619 kernel_stack_watchdog.cc:198] Thread 29611 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
21:26:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:26:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:26:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:26:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:26:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:21.291373 29611 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.765s user 0.614s sys 1.032s
21:26:21.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:21.291783 29611 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.765s user 0.614s sys 1.032s
21:26:21.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:19.527248 29621 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:21.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:21.292500 29622 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1764 milliseconds
21:26:21.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.293509 29611 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:26:21.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:21.293538 29623 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:21.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.297001 29611 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:26:21.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.299269 29611 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:26:21.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.300662 29611 hybrid_clock.cc:648] HybridClock initialized: now 1770067581300618 us; error 46 us; skew 500 ppm
21:26:21.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.301618 29611 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:26:21.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:21 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067581, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:21.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.308503 29611 init.cc:377] Logged in from keytab as kudu/127.28.70.67@KRBTEST.COM (short username kudu)
21:26:21.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.313266 29611 webserver.cc:492] Webserver started at http://127.28.70.67:35511/ using document root <none> and password file <none>
21:26:21.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.314254 29611 fs_manager.cc:362] Metadata directory not provided
21:26:21.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.314497 29611 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:26:21.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.315052 29611 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:26:21.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.319736 29611 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-2/data/instance:
21:26:21.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1091ba5cc3454b1887c6cda9c373ddaa"
21:26:21.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:21 on dist-test-slave-3trw"
21:26:21.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.320845 29611 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-2/wal/instance:
21:26:21.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1091ba5cc3454b1887c6cda9c373ddaa"
21:26:21.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:21 on dist-test-slave-3trw"
21:26:21.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.328711 29611 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.001s
21:26:21.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.335950 29635 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:26:21.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.336936 29611 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
21:26:21.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.337292 29611 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-2/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-2/wal
21:26:21.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1091ba5cc3454b1887c6cda9c373ddaa"
21:26:21.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:21 on dist-test-slave-3trw"
21:26:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.337620 29611 fs_report.cc:389] FS layout report
21:26:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:26:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-2/wal
21:26:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-2/wal
21:26:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-2/data/data
21:26:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:26:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:26:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:26:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:26:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:26:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:26:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:26:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:26:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:26:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:26:21.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:26:21.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.406888 29611 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:26:21.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.415990 29611 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:26:21.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.416512 29611 kserver.cc:163] Server-wide thread pool size limit: 3276
21:26:21.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.419461 29611 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:26:21.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.424759 29611 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:26:21.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.424960 29611 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:26:21.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.425168 29611 ts_tablet_manager.cc:616] Registered 0 tablets
21:26:21.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.425308 29611 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:26:21.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.596118 29611 rpc_server.cc:307] RPC server started. Bound to: 127.28.70.67:38533
21:26:21.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.596223 29747 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.70.67:38533 every 8 connection(s)
21:26:21.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.598691 29611 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-2/data/info.pb
21:26:21.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.601467 28953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 29611
21:26:21.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.601954 28953 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/ts-2/wal/instance
21:26:21.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:21 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067581, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.67@KRBTEST.COM for kudu/127.28.70.125@KRBTEST.COM
21:26:21.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:21 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067581, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.67@KRBTEST.COM for kudu/127.28.70.124@KRBTEST.COM
21:26:21.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:21 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067581, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.67@KRBTEST.COM for kudu/127.28.70.126@KRBTEST.COM
21:26:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.667464 29749 heartbeater.cc:344] Connected to a master server at 127.28.70.125:34925
21:26:21.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.667984 29749 heartbeater.cc:461] Registering TS with master...
21:26:21.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.669476 29749 heartbeater.cc:507] Master 127.28.70.125:34925 requested a full tablet report, sending...
21:26:21.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.671938 29748 heartbeater.cc:344] Connected to a master server at 127.28.70.124:42699
21:26:21.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.672274 29748 heartbeater.cc:461] Registering TS with master...
21:26:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.673249 29748 heartbeater.cc:507] Master 127.28.70.124:42699 requested a full tablet report, sending...
21:26:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.673329 29099 ts_manager.cc:194] Registered new tserver with Master: 1091ba5cc3454b1887c6cda9c373ddaa (127.28.70.67:38533)
21:26:21.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.675854 29190 ts_manager.cc:194] Registered new tserver with Master: 1091ba5cc3454b1887c6cda9c373ddaa (127.28.70.67:38533)
21:26:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.680768 29753 heartbeater.cc:344] Connected to a master server at 127.28.70.126:45535
21:26:21.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.681020 29753 heartbeater.cc:461] Registering TS with master...
21:26:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.681810 29753 heartbeater.cc:507] Master 127.28.70.126:45535 requested a full tablet report, sending...
21:26:21.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.683512 29015 ts_manager.cc:194] Registered new tserver with Master: 1091ba5cc3454b1887c6cda9c373ddaa (127.28.70.67:38533)
21:26:21.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.684682 29015 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.28.70.67@KRBTEST.COM'} at 127.28.70.67:43361
21:26:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.696470 28953 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
21:26:21.709 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:21.710 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
21:26:21.723 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
21:26:21.723 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
21:26:21.726 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.28.70.126"
port: 45535
}
bound_http_address {
host: "127.28.70.126"
port: 46269
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.28.70.125"
port: 34925
}
bound_http_address {
host: "127.28.70.125"
port: 38211
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.28.70.124"
port: 42699
}
bound_http_address {
host: "127.28.70.124"
port: 34373
}
}
}
21:26:21.726 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:26:21.727 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.28.70.65"
port: 44823
}
bound_http_address {
host: "127.28.70.65"
port: 41911
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.28.70.66"
port: 40871
}
bound_http_address {
host: "127.28.70.66"
port: 43623
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.28.70.67"
port: 38533
}
bound_http_address {
host: "127.28.70.67"
port: 35511
}
}
}
21:26:21.727 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:26:21.729 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
21:26:21.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:21 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067581, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:21.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:26:21.764 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:21.766 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc
21:26:21.772 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:26:21.773 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc
21:26:21.774 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:26:21.778 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
21:26:21.780 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:26:21.780 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
21:26:21.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:21 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067581, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:21.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:26:21.821 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:21.822 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc
21:26:21.823 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:26:21.823 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.70.126:45535
21:26:21.825 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
21:26:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.826484 28953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 28982
21:26:21.863 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:21.863 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.70.125:34925
21:26:21.863 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
21:26:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.864112 28953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 29060
21:26:21.898 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:21.898 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.70.124:42699
21:26:21.899 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:26:21.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.899330 28953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 29143
21:26:21.927 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:21.927 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.70.126:45535
21:26:21.931 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
21:26:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:21.932626 28953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/logs
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data/info.pb
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:26:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:26:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.126:45535
21:26:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.126
21:26:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46269
21:26:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.126
21:26:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:26:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:26:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:26:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:21.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:21.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:21.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:26:21.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:26:22.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:22.071822 29598 heartbeater.cc:646] Failed to heartbeat to 127.28.70.125:34925 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.28.70.125:34925: connect: Connection refused (error 111)
21:26:22.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:22.260462 29762 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:26:22.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:22.261081 29762 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:26:22.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:22.261695 29762 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:26:22.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:22.294804 29762 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:26:22.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:22.295023 29762 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:26:22.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:22.295348 29762 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:26:22.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:22.295554 29762 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:26:22.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:22.295743 29762 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:26:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:22.295933 29762 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:26:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:22.296082 29762 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45551/jwks.json
21:26:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:22.333287 29762 master_runner.cc:386] Master server non-default flags:
21:26:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data
21:26:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.126
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.126:45535
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data/info.pb
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.126
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46269
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/logs
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:26:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:26:22.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:26:22.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:22.334579 29762 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:26:22.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:22.335538 29762 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:26:22.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:22.347460 29769 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:22.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:22.399058 29449 heartbeater.cc:646] Failed to heartbeat to 127.28.70.125:34925 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.28.70.125:34925: connect: Connection refused (error 111)
21:26:22.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:22.688055 29753 heartbeater.cc:499] Master 127.28.70.126:45535 was elected leader, sending a full tablet report...
21:26:22.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:22.695081 29749 heartbeater.cc:646] Failed to heartbeat to 127.28.70.125:34925 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.28.70.125:34925: connect: Connection refused (error 111)
21:26:23.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:22.348853 29770 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:23.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:22.351100 29772 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:23.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:23.774151 29771 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1421 milliseconds
21:26:23.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.774312 29762 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:26:23.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.775483 29762 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:26:23.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.777989 29762 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:26:23.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.779311 29762 hybrid_clock.cc:648] HybridClock initialized: now 1770067583779262 us; error 57 us; skew 500 ppm
21:26:23.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.780182 29762 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:26:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:23 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067583, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:23.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.786334 29762 init.cc:377] Logged in from keytab as kudu/127.28.70.126@KRBTEST.COM (short username kudu)
21:26:23.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.792266 29762 webserver.cc:492] Webserver started at http://127.28.70.126:46269/ using document root <none> and password file <none>
21:26:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.793113 29762 fs_manager.cc:362] Metadata directory not provided
21:26:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.793308 29762 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:26:23.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.801584 29762 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
21:26:23.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.806504 29792 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:26:23.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.807786 29762 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
21:26:23.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.808195 29762 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "060798801f6146ffa33df46797fa9503"
21:26:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:06 on dist-test-slave-3trw"
21:26:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.810935 29762 fs_report.cc:389] FS layout report
21:26:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:26:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data/data
21:26:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:26:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:26:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:26:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:26:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:26:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:26:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:26:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:26:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:26:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:26:23.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:26:23.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.883462 29762 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:26:23.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.889009 29762 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:26:23.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.889428 29762 kserver.cc:163] Server-wide thread pool size limit: 3276
21:26:23.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.967383 29762 rpc_server.cc:307] RPC server started. Bound to: 127.28.70.126:45535
21:26:23.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.967434 29843 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.70.126:45535 every 8 connection(s)
21:26:23.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.971051 29762 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data/info.pb
21:26:23.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.977977 28953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 29762
21:26:23.978 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:23.979 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.70.125:34925
21:26:23.979 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
21:26:23.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.981208 28953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:23.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:23.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:23.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data
21:26:23.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:26:23.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:26:23.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/logs
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data/info.pb
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.125:34925
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.125
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38211
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.125
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:26:23.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:26:23.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:23.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:23.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:23.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:23.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:23.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:23.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:23.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:23.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:26:23.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:26:23.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.986639 29844 sys_catalog.cc:263] Verifying existing consensus state
21:26:23.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:23.992939 29844 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Bootstrap starting.
21:26:24.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.031133 29844 log.cc:826] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Log is configured to *not* fsync() on all Append() calls
21:26:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.046903 29844 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:26:24.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.047753 29844 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Bootstrap complete.
21:26:24.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.070214 29844 raft_consensus.cc:359] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:24.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.072355 29844 raft_consensus.cc:740] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 060798801f6146ffa33df46797fa9503, State: Initialized, Role: FOLLOWER
21:26:24.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.073177 29844 consensus_queue.cc:260] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:24.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.075850 29848 sys_catalog.cc:455] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:24.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.076900 29844 sys_catalog.cc:565] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: configured and running, proceeding with master startup.
21:26:24.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.076900 29848 sys_catalog.cc:458] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: This master's current role is: FOLLOWER
21:26:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.123530 29862 catalog_manager.cc:1269] Loaded cluster ID: 1d6e92849c0c462dbb1161273536d9ce
21:26:24.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.123804 29862 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: loading cluster ID for follower catalog manager: success
21:26:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.131979 29862 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: acquiring CA information for follower catalog manager: success
21:26:24.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.137090 29862 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:26:24.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.140641 29599 heartbeater.cc:344] Connected to a master server at 127.28.70.126:45535
21:26:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:24 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067583, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.126@KRBTEST.COM for kudu/127.28.70.126@KRBTEST.COM
21:26:24.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.148849 29809 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "56b758d0c850420b8576cdd41f6d9658" instance_seqno: 1770067578937003) as {username='kudu', principal='kudu/127.28.70.66@KRBTEST.COM'} at 127.28.70.66:43905; Asking this server to re-register.
21:26:24.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.150416 29599 heartbeater.cc:461] Registering TS with master...
21:26:24.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.154253 29809 ts_manager.cc:194] Registered new tserver with Master: 56b758d0c850420b8576cdd41f6d9658 (127.28.70.66:40871)
21:26:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.162706 29762 master_runner.cc:186] Error getting master registration for 127.28.70.125:34925: Network error: Client connection negotiation failed: client connection to 127.28.70.125:34925: connect: Connection refused (error 111),
21:26:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.189471 29762 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:26:24.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:24.344369 29846 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:26:24.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:24.345037 29846 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:26:24.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:24.345778 29846 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:26:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:24.380959 29846 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:26:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:24.381273 29846 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:26:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:24.381673 29846 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:26:24.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:24.382009 29846 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:26:24.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:24.382340 29846 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:26:24.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:24.382650 29846 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:26:24.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:24.382889 29846 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45551/jwks.json
21:26:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.436060 29846 master_runner.cc:386] Master server non-default flags:
21:26:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data
21:26:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:26:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:24.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:24.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:24.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.125
21:26:24.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:24.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:24.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:24.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.125:34925
21:26:24.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:24.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data/info.pb
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.125
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38211
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/logs
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.437626 29450 heartbeater.cc:344] Connected to a master server at 127.28.70.126:45535
21:26:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.437754 29846 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:26:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.439112 29846 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:26:24.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.442883 29809 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "7de4d9a040ba4467892670bc54c36b8c" instance_seqno: 1770067576174484) as {username='kudu', principal='kudu/127.28.70.65@KRBTEST.COM'} at 127.28.70.65:54195; Asking this server to re-register.
21:26:24.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.444357 29450 heartbeater.cc:461] Registering TS with master...
21:26:24.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.447208 29809 ts_manager.cc:194] Registered new tserver with Master: 7de4d9a040ba4467892670bc54c36b8c (127.28.70.65:44823)
21:26:24.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:24.455034 29874 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:24.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.754278 29753 heartbeater.cc:344] Connected to a master server at 127.28.70.126:45535
21:26:24.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.754779 29753 heartbeater.cc:499] Master 127.28.70.126:45535 was elected leader, sending a full tablet report...
21:26:24.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.757313 29809 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1091ba5cc3454b1887c6cda9c373ddaa" instance_seqno: 1770067581557030) as {username='kudu', principal='kudu/127.28.70.67@KRBTEST.COM'} at 127.28.70.67:55533; Asking this server to re-register.
21:26:24.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.760078 29753 heartbeater.cc:461] Registering TS with master...
21:26:24.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:24.769886 29809 ts_manager.cc:194] Registered new tserver with Master: 1091ba5cc3454b1887c6cda9c373ddaa (127.28.70.67:38533)
21:26:25.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:25 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067585, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:25.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:25.459414 29329 init.cc:295] Successfully reacquired a new kerberos TGT
21:26:25.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:25.723136 29885 raft_consensus.cc:493] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:26:25.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:25.723977 29885 raft_consensus.cc:515] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:25.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:25.741619 29885 leader_election.cc:290] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers c07e7962716a40a8bb9a84004a578cfb (127.28.70.125:34925), 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699)
21:26:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:25.746642 29795 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:25.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:25.790030 29795 leader_election.cc:336] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699): Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:25.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:25.793758 29794 leader_election.cc:336] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer c07e7962716a40a8bb9a84004a578cfb (127.28.70.125:34925): Network error: Client connection negotiation failed: client connection to 127.28.70.125:34925: connect: Connection refused (error 111)
21:26:25.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:25.794492 29794 leader_election.cc:304] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 060798801f6146ffa33df46797fa9503; no voters: 0da549b46dae45938c88c7a89dae6e27, c07e7962716a40a8bb9a84004a578cfb
21:26:25.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:25.795672 29885 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
21:26:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:24.455368 29875 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:26.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:25.856810 29873 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 29846
21:26:26.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:26.228828 29846 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.776s user 0.614s sys 1.067s
21:26:26.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:26.232110 29846 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.779s user 0.614s sys 1.067s
21:26:26.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.245136 29846 server_base.cc:1060] running on GCE node
21:26:26.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:26.246026 29894 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:26.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.247325 29846 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:26:26.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.250123 29846 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:26:26.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.251483 29846 hybrid_clock.cc:648] HybridClock initialized: now 1770067586251431 us; error 54 us; skew 500 ppm
21:26:26.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.252400 29846 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:26:26.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:26 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067586, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.259346 29846 init.cc:377] Logged in from keytab as kudu/127.28.70.125@KRBTEST.COM (short username kudu)
21:26:26.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.265106 29846 webserver.cc:492] Webserver started at http://127.28.70.125:38211/ using document root <none> and password file <none>
21:26:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.266342 29846 fs_manager.cc:362] Metadata directory not provided
21:26:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.266649 29846 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:26:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.276955 29846 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.001s
21:26:26.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.282732 29900 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:26:26.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.284011 29846 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.002s
21:26:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.284401 29846 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c07e7962716a40a8bb9a84004a578cfb"
21:26:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:08 on dist-test-slave-3trw"
21:26:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.286383 29846 fs_report.cc:389] FS layout report
21:26:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:26:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:26.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:26.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data/data
21:26:26.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:26:26.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:26:26.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:26:26.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:26:26.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:26:26.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:26:26.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:26:26.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:26:26.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:26:26.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:26:26.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:26:26.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.351791 29846 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:26:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.358816 29846 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:26:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.359333 29846 kserver.cc:163] Server-wide thread pool size limit: 3276
21:26:26.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.441587 29846 rpc_server.cc:307] RPC server started. Bound to: 127.28.70.125:34925
21:26:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.441712 29951 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.70.125:34925 every 8 connection(s)
21:26:26.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.445384 29846 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data/info.pb
21:26:26.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.450366 28953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 29846
21:26:26.451 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:26.451 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.70.124:42699
21:26:26.452 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
21:26:26.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.454523 28953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:26.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:26.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:26.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data
21:26:26.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:26:26.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:26:26.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:26:26.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:26.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:26.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/logs
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data/info.pb
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.124:42699
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.124
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34373
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.124
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:26:26.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:26:26.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.460146 29952 sys_catalog.cc:263] Verifying existing consensus state
21:26:26.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.469861 29952 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: Bootstrap starting.
21:26:26.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:26 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067585, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.65@KRBTEST.COM for kudu/127.28.70.125@KRBTEST.COM
21:26:26.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.531548 29952 log.cc:826] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: Log is configured to *not* fsync() on all Append() calls
21:26:26.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.540293 29449 heartbeater.cc:344] Connected to a master server at 127.28.70.125:34925
21:26:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.561579 29952 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:26:26.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.562633 29952 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: Bootstrap complete.
21:26:26.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.592228 29952 raft_consensus.cc:359] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:26.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.595468 29952 raft_consensus.cc:740] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: c07e7962716a40a8bb9a84004a578cfb, State: Initialized, Role: FOLLOWER
21:26:26.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.596357 29952 consensus_queue.cc:260] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:26.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.615041 29959 sys_catalog.cc:455] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:26.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.616364 29959 sys_catalog.cc:458] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: This master's current role is: FOLLOWER
21:26:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.617630 29952 sys_catalog.cc:565] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: configured and running, proceeding with master startup.
21:26:26.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.676687 29970 catalog_manager.cc:1269] Loaded cluster ID: 1d6e92849c0c462dbb1161273536d9ce
21:26:26.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.677276 29970 catalog_manager.cc:1562] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: loading cluster ID for follower catalog manager: success
21:26:26.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.690680 29970 catalog_manager.cc:1584] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: acquiring CA information for follower catalog manager: success
21:26:26.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.698980 29970 catalog_manager.cc:1612] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:26:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:26 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067586, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.125@KRBTEST.COM for kudu/127.28.70.126@KRBTEST.COM
21:26:26.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.790937 29749 heartbeater.cc:344] Connected to a master server at 127.28.70.125:34925
21:26:26.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.762868 29846 master_runner.cc:186] Error getting master registration for 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111),
21:26:26.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.800642 29846 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:26:26.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.803202 29917 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1091ba5cc3454b1887c6cda9c373ddaa" instance_seqno: 1770067581557030) as {username='kudu', principal='kudu/127.28.70.67@KRBTEST.COM'} at 127.28.70.67:35753; Asking this server to re-register.
21:26:26.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.804829 29749 heartbeater.cc:461] Registering TS with master...
21:26:26.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:26.808511 29916 ts_manager.cc:194] Registered new tserver with Master: 1091ba5cc3454b1887c6cda9c373ddaa (127.28.70.67:38533)
21:26:26.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:26.936861 29954 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:26:26.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:26.937630 29954 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:26:26.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:26.938721 29954 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:26:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:27.000850 29954 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:26:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:27.001204 29954 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:26:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:27.001698 29954 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:26:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:27.002067 29954 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:26:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:27.002424 29954 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:26:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:27.002759 29954 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:26:27.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:27.003017 29954 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45551/jwks.json
21:26:27.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.068114 29954 master_runner.cc:386] Master server non-default flags:
21:26:27.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:27.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:27.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:27.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:27.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data
21:26:27.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:27.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:26:27.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:27.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:27.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:27.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.124
21:26:27.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:27.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:27.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:27.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.124:42699
21:26:27.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data/info.pb
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.124
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34373
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/logs
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.069348 29954 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:26:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.070564 29954 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:26:27.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:27.084448 29983 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.170950 29598 heartbeater.cc:344] Connected to a master server at 127.28.70.125:34925
21:26:27.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.174628 29917 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "56b758d0c850420b8576cdd41f6d9658" instance_seqno: 1770067578937003) as {username='kudu', principal='kudu/127.28.70.66@KRBTEST.COM'} at 127.28.70.66:48357; Asking this server to re-register.
21:26:27.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.176036 29598 heartbeater.cc:461] Registering TS with master...
21:26:27.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.179208 29917 ts_manager.cc:194] Registered new tserver with Master: 56b758d0c850420b8576cdd41f6d9658 (127.28.70.66:40871)
21:26:27.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.555435 29917 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "7de4d9a040ba4467892670bc54c36b8c" instance_seqno: 1770067576174484) as {username='kudu', principal='kudu/127.28.70.65@KRBTEST.COM'} at 127.28.70.65:46797; Asking this server to re-register.
21:26:27.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.558779 29449 heartbeater.cc:461] Registering TS with master...
21:26:27.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.562296 29917 ts_manager.cc:194] Registered new tserver with Master: 7de4d9a040ba4467892670bc54c36b8c (127.28.70.65:44823)
21:26:27.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.658212 29991 raft_consensus.cc:493] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:26:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.659077 29991 raft_consensus.cc:515] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:27.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.676764 29991 leader_election.cc:290] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers c07e7962716a40a8bb9a84004a578cfb (127.28.70.125:34925), 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699)
21:26:27.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.696971 29927 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "060798801f6146ffa33df46797fa9503" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "c07e7962716a40a8bb9a84004a578cfb" is_pre_election: true
21:26:27.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:27.697378 29795 leader_election.cc:336] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699): Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:27.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.697930 29927 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 060798801f6146ffa33df46797fa9503 in term 1.
21:26:27.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.699421 29794 leader_election.cc:304] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 060798801f6146ffa33df46797fa9503, c07e7962716a40a8bb9a84004a578cfb; no voters: 0da549b46dae45938c88c7a89dae6e27
21:26:27.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.701202 29991 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 1 FOLLOWER]: Leader pre-election won for term 2
21:26:27.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.701678 29991 raft_consensus.cc:493] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:26:27.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.702165 29991 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 1 FOLLOWER]: Advancing to term 2
21:26:27.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.718818 29991 raft_consensus.cc:515] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.724511 29927 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "060798801f6146ffa33df46797fa9503" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "c07e7962716a40a8bb9a84004a578cfb"
21:26:27.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.725389 29927 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 1 FOLLOWER]: Advancing to term 2
21:26:27.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.740002 29991 leader_election.cc:290] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 2 election: Requested vote from peers c07e7962716a40a8bb9a84004a578cfb (127.28.70.125:34925), 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699)
21:26:27.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.742641 29927 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 060798801f6146ffa33df46797fa9503 in term 2.
21:26:27.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:27.744153 29795 leader_election.cc:336] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699): Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:27.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.744832 29795 leader_election.cc:304] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 060798801f6146ffa33df46797fa9503, c07e7962716a40a8bb9a84004a578cfb; no voters: 0da549b46dae45938c88c7a89dae6e27
21:26:27.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.750072 29991 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 2 FOLLOWER]: Leader election won for term 2
21:26:27.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.755177 29991 raft_consensus.cc:697] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 2 LEADER]: Becoming Leader. State: Replica: 060798801f6146ffa33df46797fa9503, State: Running, Role: LEADER
21:26:27.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.756414 29991 consensus_queue.cc:237] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:27.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.771610 29995 sys_catalog.cc:455] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 060798801f6146ffa33df46797fa9503. Latest consensus state: current_term: 2 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:27.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.772622 29995 sys_catalog.cc:458] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: This master's current role is: LEADER
21:26:27.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.784411 29997 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:26:27.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.811503 29997 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:26:27.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.865440 29997 catalog_manager.cc:1269] Loaded cluster ID: 1d6e92849c0c462dbb1161273536d9ce
21:26:27.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.865715 29997 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:26:27.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.878506 29997 catalog_manager.cc:1514] Loading token signing keys...
21:26:27.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.885265 29997 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Loaded TSK: 0
21:26:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:27.895901 29997 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:26:28.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.227046 29927 raft_consensus.cc:1275] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 2 FOLLOWER]: Refusing update from remote peer 060798801f6146ffa33df46797fa9503: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
21:26:28.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.228705 29995 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
21:26:28.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.249527 30001 sys_catalog.cc:455] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: SysCatalogTable state changed. Reason: New leader 060798801f6146ffa33df46797fa9503. Latest consensus state: current_term: 2 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:28.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.250548 30001 sys_catalog.cc:458] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: This master's current role is: FOLLOWER
21:26:28.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.282308 30001 sys_catalog.cc:455] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:28.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.283061 30001 sys_catalog.cc:458] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: This master's current role is: FOLLOWER
21:26:28.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.283061 29995 sys_catalog.cc:455] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:28.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.283727 29995 sys_catalog.cc:458] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: This master's current role is: LEADER
21:26:28.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:28.289469 29795 consensus_peers.cc:597] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 -> Peer 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699): Couldn't send request to peer 0da549b46dae45938c88c7a89dae6e27. Status: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:26:28.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:28 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067588, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:28.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.615320 29480 init.cc:295] Successfully reacquired a new kerberos TGT
21:26:28.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:27.087321 29984 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:28.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:28.487490 29982 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 29954
21:26:28.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:28.794893 29954 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.709s user 0.646s sys 0.919s
21:26:28.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:28.796888 29954 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.711s user 0.647s sys 0.919s
21:26:28.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:28.812160 30009 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:28.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.818853 29954 server_base.cc:1060] running on GCE node
21:26:28.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.820935 29954 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:26:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.834530 29954 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:26:28.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.835980 29954 hybrid_clock.cc:648] HybridClock initialized: now 1770067588835938 us; error 49 us; skew 500 ppm
21:26:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.837033 29954 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:26:28.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:28 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067588, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.845973 29954 init.cc:377] Logged in from keytab as kudu/127.28.70.124@KRBTEST.COM (short username kudu)
21:26:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.883802 29954 webserver.cc:492] Webserver started at http://127.28.70.124:34373/ using document root <none> and password file <none>
21:26:28.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.884891 29954 fs_manager.cc:362] Metadata directory not provided
21:26:28.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.885205 29954 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:26:28.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.894557 29954 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
21:26:28.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.900015 30016 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:26:28.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.901304 29954 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
21:26:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.901723 29954 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0da549b46dae45938c88c7a89dae6e27"
21:26:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:11 on dist-test-slave-3trw"
21:26:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.904313 29954 fs_report.cc:389] FS layout report
21:26:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:26:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data/data
21:26:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:26:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:26:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:26:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:26:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:26:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:26:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:26:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:26:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:26:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:26:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:26:28.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.980713 29954 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:26:28.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.989652 29954 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:26:28.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:28.990274 29954 kserver.cc:163] Server-wide thread pool size limit: 3276
21:26:29.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.086172 29954 rpc_server.cc:307] RPC server started. Bound to: 127.28.70.124:42699
21:26:29.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.086205 30067 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.70.124:42699 every 8 connection(s)
21:26:29.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.090059 29954 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data/info.pb
21:26:29.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.090652 28953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 29954
21:26:29.091 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:29.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.104099 30068 sys_catalog.cc:263] Verifying existing consensus state
21:26:29.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.109934 30068 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: Bootstrap starting.
21:26:29.150 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.70.126:45535(127.28.70.126:45535)] connecting to peer
21:26:29.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.165323 30068 log.cc:826] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: Log is configured to *not* fsync() on all Append() calls
21:26:29.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.173974 29599 heartbeater.cc:499] Master 127.28.70.126:45535 was elected leader, sending a full tablet report...
21:26:29.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.195381 30068 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:26:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.196436 30068 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: Bootstrap complete.
21:26:29.228 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.70.125:34925(127.28.70.125:34925)] connecting to peer
21:26:29.229 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.70.124:42699(127.28.70.124:42699)] connecting to peer
21:26:29.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.235481 30068 raft_consensus.cc:359] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:29.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.238893 30068 raft_consensus.cc:740] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0da549b46dae45938c88c7a89dae6e27, State: Initialized, Role: FOLLOWER
21:26:29.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:29 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067588, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.66@KRBTEST.COM for kudu/127.28.70.124@KRBTEST.COM
21:26:29.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.243404 30068 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:29.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:29 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067583, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.126@KRBTEST.COM for kudu/127.28.70.124@KRBTEST.COM
21:26:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.271286 29762 master_runner.cc:186] Error getting master registration for 127.28.70.124:42699: OK, instance_id { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" instance_seqno: 1770067589062656 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:26:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.272193 29762 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:26:29.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.279973 30078 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:29.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.285794 29597 heartbeater.cc:344] Connected to a master server at 127.28.70.124:42699
21:26:29.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.286113 30078 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: This master's current role is: FOLLOWER
21:26:29.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.289656 30068 sys_catalog.cc:565] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: configured and running, proceeding with master startup.
21:26:29.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.290398 30043 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [term 1 FOLLOWER]: Advancing to term 2
21:26:29.305 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.28.70.125:34925(127.28.70.125:34925)] Successfully connected to peer
21:26:29.305 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.28.70.126:45535(127.28.70.126:45535)] Successfully connected to peer
21:26:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.367731 30078 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 060798801f6146ffa33df46797fa9503. Latest consensus state: current_term: 2 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.368875 30078 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: This master's current role is: FOLLOWER
21:26:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.369531 30092 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.370335 30092 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: This master's current role is: FOLLOWER
21:26:29.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.392500 30094 catalog_manager.cc:1269] Loaded cluster ID: 1d6e92849c0c462dbb1161273536d9ce
21:26:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.398524 30094 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: loading cluster ID for follower catalog manager: success
21:26:29.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.409770 30094 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: acquiring CA information for follower catalog manager: success
21:26:29.413 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.28.70.124:42699(127.28.70.124:42699)] Successfully connected to peer
21:26:29.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.421717 30094 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:26:29.460 [WARN - kudu-nio-0] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
21:26:29.496 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.28.70.125
21:26:29.506 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.28.70.126
21:26:29.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.543246 29450 heartbeater.cc:499] Master 127.28.70.126:45535 was elected leader, sending a full tablet report...
21:26:29.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.562614 29448 heartbeater.cc:344] Connected to a master server at 127.28.70.124:42699
21:26:29.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.568642 30033 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "7de4d9a040ba4467892670bc54c36b8c" instance_seqno: 1770067576174484) as {username='kudu', principal='kudu/127.28.70.65@KRBTEST.COM'} at 127.28.70.65:38199; Asking this server to re-register.
21:26:29.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.573390 29448 heartbeater.cc:461] Registering TS with master...
21:26:29.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.577641 30033 ts_manager.cc:194] Registered new tserver with Master: 7de4d9a040ba4467892670bc54c36b8c (127.28.70.65:44823)
21:26:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.651198 30101 sys_catalog.cc:455] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.651973 30101 sys_catalog.cc:458] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: This master's current role is: LEADER
21:26:29.738 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.28.70.124
21:26:29.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.883692 29753 heartbeater.cc:499] Master 127.28.70.126:45535 was elected leader, sending a full tablet report...
21:26:29.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.931155 29748 heartbeater.cc:344] Connected to a master server at 127.28.70.124:42699
21:26:29.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.934029 30033 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1091ba5cc3454b1887c6cda9c373ddaa" instance_seqno: 1770067581557030) as {username='kudu', principal='kudu/127.28.70.67@KRBTEST.COM'} at 127.28.70.67:52821; Asking this server to re-register.
21:26:29.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.935575 29748 heartbeater.cc:461] Registering TS with master...
21:26:29.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:29.943660 30033 ts_manager.cc:194] Registered new tserver with Master: 1091ba5cc3454b1887c6cda9c373ddaa (127.28.70.67:38533)
21:26:30.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:30 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067581, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.70.125@KRBTEST.COM
21:26:30.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:30 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067581, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.70.126@KRBTEST.COM
21:26:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:30 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067581, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.70.124@KRBTEST.COM
21:26:30.165 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xd3b316c6, L:/127.0.0.1:41036 - R:/127.28.70.125:34925] using SASL/GSSAPI
21:26:30.236 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xfb11a1e6, L:/127.0.0.1:56452 - R:/127.28.70.126:45535] using SASL/GSSAPI
21:26:30.280 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xccb31489, L:/127.0.0.1:48822 - R:/127.28.70.124:42699] using SASL/GSSAPI
21:26:30.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:30.291826 30033 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "56b758d0c850420b8576cdd41f6d9658" instance_seqno: 1770067578937003) as {username='kudu', principal='kudu/127.28.70.66@KRBTEST.COM'} at 127.28.70.66:44191; Asking this server to re-register.
21:26:30.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:30.293484 29597 heartbeater.cc:461] Registering TS with master...
21:26:30.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:30.296483 30033 ts_manager.cc:194] Registered new tserver with Master: 56b758d0c850420b8576cdd41f6d9658 (127.28.70.66:40871)
21:26:30.329 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:26:30.332 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1235), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.70.126:45535, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.70.125:34925, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.70.124:42699, [1168ms] Sub RPC ConnectToMaster: received response from server master-127.28.70.125:34925: OK, [1186ms] Sub RPC ConnectToMaster: received response from server master-127.28.70.126:45535: OK, deferred=null) after lookup
21:26:31.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:31.238966 28938 debug-util.cc:398] Leaking SignalData structure 0x7b08000add00 after lost signal to thread 28813
21:26:31.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:31.240245 28938 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad6a0 after lost signal to thread 28941
21:26:31.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:31 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067591, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:31.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:31.313359 29631 init.cc:295] Successfully reacquired a new kerberos TGT
21:26:31.368 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:26:31.368 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
21:26:31.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:31 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067591, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:31.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:26:31.412 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:31.414 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc
21:26:31.416 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:26:31.416 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.70.126:45535
21:26:31.416 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
21:26:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:31.417266 28953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 29762
21:26:31.468 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.28.70.126:45535(127.28.70.126:45535)] handling channelInactive
21:26:31.468 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.70.126:45535(127.28.70.126:45535)] cleaning up while in state READY due to: connection closed
21:26:31.475 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:31.475 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.70.125:34925
21:26:31.475 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
21:26:31.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:31.475800 28953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 29846
21:26:31.518 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.28.70.125:34925(127.28.70.125:34925)] handling channelInactive
21:26:31.519 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.70.125:34925(127.28.70.125:34925)] cleaning up while in state READY due to: connection closed
21:26:31.520 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:31.520 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.70.124:42699
21:26:31.520 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:26:31.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:31.521107 28953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 29954
21:26:31.551 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.28.70.124:42699(127.28.70.124:42699)] handling channelInactive
21:26:31.551 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.70.124:42699(127.28.70.124:42699)] cleaning up while in state READY due to: connection closed
21:26:31.552 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:31.552 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.70.126:45535
21:26:31.552 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
21:26:31.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:31.554186 28953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:31.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:31.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:31.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data
21:26:31.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:26:31.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:26:31.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:26:31.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:31.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/logs
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data/info.pb
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.126:45535
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.126
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46269
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.126
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:26:31.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:26:31.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:31.930142 30109 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:26:31.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:31.930802 30109 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:26:31.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:31.931550 30109 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:26:31.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:31.968508 30109 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:26:31.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:31.968786 30109 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:26:31.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:31.969165 30109 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:26:31.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:31.969497 30109 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:26:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:31.969801 30109 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:26:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:31.970187 30109 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:26:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:31.970425 30109 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45551/jwks.json
21:26:32.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:32.017494 30109 master_runner.cc:386] Master server non-default flags:
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.126
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.126:45535
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data/info.pb
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.126
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46269
21:26:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:26:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:26:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:26:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/logs
21:26:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:26:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:26:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:26:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:26:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:26:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:26:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:26:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:26:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:26:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:32.019170 30109 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:26:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:32.020277 30109 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:26:32.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:32.032282 30118 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:33.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:32.041227 30119 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:33.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:32.044641 30121 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:33.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:32.044006 30109 server_base.cc:1060] running on GCE node
21:26:33.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.315688 30109 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:26:33.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.318326 30109 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:26:33.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.319697 30109 hybrid_clock.cc:648] HybridClock initialized: now 1770067593319665 us; error 47 us; skew 500 ppm
21:26:33.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.320554 30109 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:26:33.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:33 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067593, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:33.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.326648 30109 init.cc:377] Logged in from keytab as kudu/127.28.70.126@KRBTEST.COM (short username kudu)
21:26:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.332450 30109 webserver.cc:492] Webserver started at http://127.28.70.126:46269/ using document root <none> and password file <none>
21:26:33.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.333638 30109 fs_manager.cc:362] Metadata directory not provided
21:26:33.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.333931 30109 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:26:33.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.344071 30109 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.000s
21:26:33.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.349253 30136 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:26:33.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.350310 30109 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
21:26:33.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.350600 30109 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:33.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "060798801f6146ffa33df46797fa9503"
21:26:33.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:06 on dist-test-slave-3trw"
21:26:33.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.352512 30109 fs_report.cc:389] FS layout report
21:26:33.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:26:33.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:33.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:33.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data/data
21:26:33.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:26:33.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:26:33.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:26:33.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:26:33.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:26:33.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:26:33.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:26:33.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:26:33.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:26:33.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:26:33.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:26:33.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.409898 30109 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:26:33.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.419593 30109 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:26:33.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.420007 30109 kserver.cc:163] Server-wide thread pool size limit: 3276
21:26:33.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.494084 30109 rpc_server.cc:307] RPC server started. Bound to: 127.28.70.126:45535
21:26:33.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.494163 30187 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.70.126:45535 every 8 connection(s)
21:26:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.496682 30109 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data/info.pb
21:26:33.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.506659 28953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 30109
21:26:33.507 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:33.507 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.70.125:34925
21:26:33.507 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
21:26:33.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.507664 30188 sys_catalog.cc:263] Verifying existing consensus state
21:26:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.508963 28953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data
21:26:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:26:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:26:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:26:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/logs
21:26:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data/info.pb
21:26:33.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:33.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:26:33.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:26:33.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:26:33.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:26:33.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:33.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:33.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:33.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:26:33.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.125:34925
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.125
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38211
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.125
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:26:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:26:33.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.514159 30188 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Bootstrap starting.
21:26:33.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.569149 30188 log.cc:826] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Log is configured to *not* fsync() on all Append() calls
21:26:33.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.593679 30188 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:26:33.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.611080 30188 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Bootstrap complete.
21:26:33.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:33 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067585, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.65@KRBTEST.COM for kudu/127.28.70.126@KRBTEST.COM
21:26:33.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.633579 29450 heartbeater.cc:344] Connected to a master server at 127.28.70.126:45535
21:26:33.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.643417 30188 raft_consensus.cc:359] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:33.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.646613 30188 raft_consensus.cc:740] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 060798801f6146ffa33df46797fa9503, State: Initialized, Role: FOLLOWER
21:26:33.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.647476 30188 consensus_queue.cc:260] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:33.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.650030 30196 sys_catalog.cc:455] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:33.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.650954 30196 sys_catalog.cc:458] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: This master's current role is: FOLLOWER
21:26:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.651548 30188 sys_catalog.cc:565] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: configured and running, proceeding with master startup.
21:26:33.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.685724 30207 catalog_manager.cc:1269] Loaded cluster ID: 1d6e92849c0c462dbb1161273536d9ce
21:26:33.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.686272 30207 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: loading cluster ID for follower catalog manager: success
21:26:33.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.693632 30207 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: acquiring CA information for follower catalog manager: success
21:26:33.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:33 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067593, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.126@KRBTEST.COM for kudu/127.28.70.126@KRBTEST.COM
21:26:33.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.699512 30207 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:26:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.712365 30109 master_runner.cc:186] Error getting master registration for 127.28.70.125:34925: Network error: Client connection negotiation failed: client connection to 127.28.70.125:34925: connect: Connection refused (error 111),
21:26:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.731468 30109 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:26:33.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:33.921716 30190 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:26:33.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:33.922317 30190 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:26:33.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:33.922912 30190 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:26:33.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.941215 29753 heartbeater.cc:344] Connected to a master server at 127.28.70.126:45535
21:26:33.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.950543 30153 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1091ba5cc3454b1887c6cda9c373ddaa" instance_seqno: 1770067581557030) as {username='kudu', principal='kudu/127.28.70.67@KRBTEST.COM'} at 127.28.70.67:57401; Asking this server to re-register.
21:26:33.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.953873 29753 heartbeater.cc:461] Registering TS with master...
21:26:33.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:33.958797 30152 ts_manager.cc:194] Registered new tserver with Master: 1091ba5cc3454b1887c6cda9c373ddaa (127.28.70.67:38533)
21:26:33.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:33.976402 30190 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:26:33.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:33.976711 30190 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:26:33.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:33.977125 30190 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:26:33.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:33.977454 30190 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:26:33.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:33.977770 30190 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:26:33.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:33.978096 30190 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:26:33.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:33.978350 30190 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45551/jwks.json
21:26:34.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:34.026078 30190 master_runner.cc:386] Master server non-default flags:
21:26:34.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:34.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:34.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:34.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:34.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data
21:26:34.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:34.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:26:34.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:34.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:34.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:34.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.125
21:26:34.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.125:34925
21:26:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:26:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:26:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:26:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data/info.pb
21:26:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.125
21:26:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38211
21:26:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:26:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:34.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:26:34.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:26:34.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:34.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/logs
21:26:34.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:34.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:26:34.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:26:34.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:26:34.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:26:34.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:26:34.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:26:34.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:26:34.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:26:34.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:26:34.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:34.027724 30190 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:26:34.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:34.029011 30190 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:26:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:34.061326 30218 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:34.279335 29599 heartbeater.cc:344] Connected to a master server at 127.28.70.126:45535
21:26:34.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:34.283687 30152 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "56b758d0c850420b8576cdd41f6d9658" instance_seqno: 1770067578937003) as {username='kudu', principal='kudu/127.28.70.66@KRBTEST.COM'} at 127.28.70.66:41321; Asking this server to re-register.
21:26:34.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:34.285223 29599 heartbeater.cc:461] Registering TS with master...
21:26:34.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:34.290592 30152 ts_manager.cc:194] Registered new tserver with Master: 56b758d0c850420b8576cdd41f6d9658 (127.28.70.66:40871)
21:26:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:34.645375 30152 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "7de4d9a040ba4467892670bc54c36b8c" instance_seqno: 1770067576174484) as {username='kudu', principal='kudu/127.28.70.65@KRBTEST.COM'} at 127.28.70.65:45157; Asking this server to re-register.
21:26:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:34.646948 29450 heartbeater.cc:461] Registering TS with master...
21:26:34.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:34.650549 30152 ts_manager.cc:194] Registered new tserver with Master: 7de4d9a040ba4467892670bc54c36b8c (127.28.70.65:44823)
21:26:35.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.143994 30231 raft_consensus.cc:493] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:26:35.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.144610 30231 raft_consensus.cc:515] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:35.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.162402 30231 leader_election.cc:290] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers c07e7962716a40a8bb9a84004a578cfb (127.28.70.125:34925), 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699)
21:26:35.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:35.165477 30139 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:35.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:35.180477 30139 leader_election.cc:336] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699): Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:35.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:35.184633 30138 leader_election.cc:336] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer c07e7962716a40a8bb9a84004a578cfb (127.28.70.125:34925): Network error: Client connection negotiation failed: client connection to 127.28.70.125:34925: connect: Connection refused (error 111)
21:26:35.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.185273 30138 leader_election.cc:304] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 060798801f6146ffa33df46797fa9503; no voters: 0da549b46dae45938c88c7a89dae6e27, c07e7962716a40a8bb9a84004a578cfb
21:26:35.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.186502 30231 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
21:26:35.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:35 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067595, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:35.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.463145 29329 init.cc:295] Successfully reacquired a new kerberos TGT
21:26:35.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:35.461697 30217 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 30190
21:26:35.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:35.636884 30217 kernel_stack_watchdog.cc:198] Thread 30190 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 393ms:
21:26:35.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:26:35.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:26:35.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:26:35.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:26:35.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:34.065290 30219 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:35.643000 30190 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.578s user 0.420s sys 0.859s
21:26:35.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:35.644286 30221 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:35.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:35.648546 30220 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1583 milliseconds
21:26:35.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:35.643374 30190 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.578s user 0.420s sys 0.859s
21:26:35.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.651691 30190 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:26:35.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.656487 30190 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:26:35.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.661072 30190 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:26:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.662544 30190 hybrid_clock.cc:648] HybridClock initialized: now 1770067595662503 us; error 49 us; skew 500 ppm
21:26:35.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.663659 30190 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:26:35.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:35 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067595, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.672031 30190 init.cc:377] Logged in from keytab as kudu/127.28.70.125@KRBTEST.COM (short username kudu)
21:26:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.677982 30190 webserver.cc:492] Webserver started at http://127.28.70.125:38211/ using document root <none> and password file <none>
21:26:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.679205 30190 fs_manager.cc:362] Metadata directory not provided
21:26:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.679479 30190 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:26:35.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.690217 30190 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.004s sys 0.003s
21:26:35.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.696233 30243 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:26:35.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.697561 30190 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
21:26:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.697975 30190 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c07e7962716a40a8bb9a84004a578cfb"
21:26:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:08 on dist-test-slave-3trw"
21:26:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.700611 30190 fs_report.cc:389] FS layout report
21:26:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:26:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data/data
21:26:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:26:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:26:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:26:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:26:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:26:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:26:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:26:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:26:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:26:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:26:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:26:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.779608 30190 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:26:35.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.787127 30190 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:26:35.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.787688 30190 kserver.cc:163] Server-wide thread pool size limit: 3276
21:26:35.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.892174 30190 rpc_server.cc:307] RPC server started. Bound to: 127.28.70.125:34925
21:26:35.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.892292 30294 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.70.125:34925 every 8 connection(s)
21:26:35.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.897540 30190 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data/info.pb
21:26:35.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.907001 28953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 30190
21:26:35.909 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:35.909 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.70.124:42699
21:26:35.909 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
21:26:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.911322 28953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/logs
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data/info.pb
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.124:42699
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.124
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34373
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.124
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:35.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:26:35.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:26:35.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.912168 30295 sys_catalog.cc:263] Verifying existing consensus state
21:26:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.927323 30295 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: Bootstrap starting.
21:26:35.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:35 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067591, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.67@KRBTEST.COM for kudu/127.28.70.125@KRBTEST.COM
21:26:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:35.973917 29749 heartbeater.cc:344] Connected to a master server at 127.28.70.125:34925
21:26:36.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.001696 30295 log.cc:826] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: Log is configured to *not* fsync() on all Append() calls
21:26:36.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.020605 30295 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:26:36.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.021476 30295 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: Bootstrap complete.
21:26:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.048385 30295 raft_consensus.cc:359] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.051803 30295 raft_consensus.cc:740] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: c07e7962716a40a8bb9a84004a578cfb, State: Initialized, Role: FOLLOWER
21:26:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.052749 30295 consensus_queue.cc:260] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:36.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.056031 30302 sys_catalog.cc:455] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:36.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.057016 30302 sys_catalog.cc:458] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: This master's current role is: FOLLOWER
21:26:36.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.058277 30295 sys_catalog.cc:565] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: configured and running, proceeding with master startup.
21:26:36.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.117972 30313 catalog_manager.cc:1269] Loaded cluster ID: 1d6e92849c0c462dbb1161273536d9ce
21:26:36.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.118549 30313 catalog_manager.cc:1562] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: loading cluster ID for follower catalog manager: success
21:26:36.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:36 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067595, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.125@KRBTEST.COM for kudu/127.28.70.126@KRBTEST.COM
21:26:36.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.134466 30313 catalog_manager.cc:1584] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: acquiring CA information for follower catalog manager: success
21:26:36.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.147188 30313 catalog_manager.cc:1612] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:26:36.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.190977 30190 master_runner.cc:186] Error getting master registration for 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111),
21:26:36.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.217190 30190 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:26:36.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.267472 29598 heartbeater.cc:344] Connected to a master server at 127.28.70.125:34925
21:26:36.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.273303 30260 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "56b758d0c850420b8576cdd41f6d9658" instance_seqno: 1770067578937003) as {username='kudu', principal='kudu/127.28.70.66@KRBTEST.COM'} at 127.28.70.66:53287; Asking this server to re-register.
21:26:36.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.275063 29598 heartbeater.cc:461] Registering TS with master...
21:26:36.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.278049 30260 ts_manager.cc:194] Registered new tserver with Master: 56b758d0c850420b8576cdd41f6d9658 (127.28.70.66:40871)
21:26:36.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:36.442273 30299 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:26:36.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:36.443099 30299 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:26:36.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:36.444221 30299 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:26:36.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:36.502519 30299 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:26:36.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:36.502856 30299 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:26:36.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:36.503343 30299 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:26:36.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:36.503698 30299 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:26:36.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:36.504035 30299 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:26:36.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:36.504379 30299 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:26:36.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:36.504631 30299 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45551/jwks.json
21:26:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.570912 30299 master_runner.cc:386] Master server non-default flags:
21:26:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data
21:26:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:26:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.124
21:26:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.124:42699
21:26:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:26:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:26:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:26:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data/info.pb
21:26:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.124
21:26:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34373
21:26:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:26:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:26:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:26:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/logs
21:26:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:26:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:26:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:26:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:26:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:26:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:26:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:26:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:26:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:26:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.572563 30299 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:26:36.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.573951 30299 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:26:36.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:36.589193 30324 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:36.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.693866 29449 heartbeater.cc:344] Connected to a master server at 127.28.70.125:34925
21:26:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.696722 30260 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "7de4d9a040ba4467892670bc54c36b8c" instance_seqno: 1770067576174484) as {username='kudu', principal='kudu/127.28.70.65@KRBTEST.COM'} at 127.28.70.65:52413; Asking this server to re-register.
21:26:36.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.698351 29449 heartbeater.cc:461] Registering TS with master...
21:26:36.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.701321 30260 ts_manager.cc:194] Registered new tserver with Master: 7de4d9a040ba4467892670bc54c36b8c (127.28.70.65:44823)
21:26:36.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.913985 30333 raft_consensus.cc:493] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:26:36.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.914484 30333 raft_consensus.cc:515] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:36.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.921491 30333 leader_election.cc:290] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers c07e7962716a40a8bb9a84004a578cfb (127.28.70.125:34925), 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699)
21:26:36.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.982429 30260 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1091ba5cc3454b1887c6cda9c373ddaa" instance_seqno: 1770067581557030) as {username='kudu', principal='kudu/127.28.70.67@KRBTEST.COM'} at 127.28.70.67:58821; Asking this server to re-register.
21:26:36.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.985306 29749 heartbeater.cc:461] Registering TS with master...
21:26:36.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:36.990983 30260 ts_manager.cc:194] Registered new tserver with Master: 1091ba5cc3454b1887c6cda9c373ddaa (127.28.70.67:38533)
21:26:37.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:37.012355 30139 leader_election.cc:336] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699): Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:37.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.016646 30270 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "060798801f6146ffa33df46797fa9503" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "c07e7962716a40a8bb9a84004a578cfb" is_pre_election: true
21:26:37.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.017580 30270 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 060798801f6146ffa33df46797fa9503 in term 2.
21:26:37.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.025987 30138 leader_election.cc:304] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 060798801f6146ffa33df46797fa9503, c07e7962716a40a8bb9a84004a578cfb; no voters: 0da549b46dae45938c88c7a89dae6e27
21:26:37.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.030361 30333 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 2 FOLLOWER]: Leader pre-election won for term 3
21:26:37.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.030714 30333 raft_consensus.cc:493] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:26:37.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.031002 30333 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 2 FOLLOWER]: Advancing to term 3
21:26:37.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.044660 30333 raft_consensus.cc:515] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:37.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.052210 30270 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "060798801f6146ffa33df46797fa9503" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "c07e7962716a40a8bb9a84004a578cfb"
21:26:37.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.052802 30270 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 2 FOLLOWER]: Advancing to term 3
21:26:37.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.064818 30270 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 060798801f6146ffa33df46797fa9503 in term 3.
21:26:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.069649 30138 leader_election.cc:304] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 060798801f6146ffa33df46797fa9503, c07e7962716a40a8bb9a84004a578cfb; no voters:
21:26:37.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:37.071228 30139 leader_election.cc:336] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699): Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:37.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.048161 30333 leader_election.cc:290] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 3 election: Requested vote from peers c07e7962716a40a8bb9a84004a578cfb (127.28.70.125:34925), 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699)
21:26:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.074820 30339 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 3 FOLLOWER]: Leader election won for term 3
21:26:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.076547 30339 raft_consensus.cc:697] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 3 LEADER]: Becoming Leader. State: Replica: 060798801f6146ffa33df46797fa9503, State: Running, Role: LEADER
21:26:37.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.079203 30339 consensus_queue.cc:237] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.098704 30333 sys_catalog.cc:455] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 060798801f6146ffa33df46797fa9503. Latest consensus state: current_term: 3 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.099699 30333 sys_catalog.cc:458] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: This master's current role is: LEADER
21:26:37.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.113969 30341 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:26:37.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.124429 30341 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:26:37.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.143153 30341 catalog_manager.cc:1269] Loaded cluster ID: 1d6e92849c0c462dbb1161273536d9ce
21:26:37.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.144806 30341 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:26:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.157748 30341 catalog_manager.cc:1514] Loading token signing keys...
21:26:37.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.169258 30341 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Loaded TSK: 0
21:26:37.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.172963 30341 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:26:37.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.473562 30270 raft_consensus.cc:1275] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 3 FOLLOWER]: Refusing update from remote peer 060798801f6146ffa33df46797fa9503: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
21:26:37.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.476769 30333 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
21:26:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.532147 30344 sys_catalog.cc:455] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: SysCatalogTable state changed. Reason: New leader 060798801f6146ffa33df46797fa9503. Latest consensus state: current_term: 3 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:37.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.533125 30344 sys_catalog.cc:458] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: This master's current role is: FOLLOWER
21:26:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.547077 30344 sys_catalog.cc:455] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:37.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.548007 30344 sys_catalog.cc:458] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: This master's current role is: FOLLOWER
21:26:37.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:37.557377 30139 consensus_peers.cc:597] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 -> Peer 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699): Couldn't send request to peer 0da549b46dae45938c88c7a89dae6e27. Status: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:26:37.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.559429 30339 sys_catalog.cc:455] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:37.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:37.560149 30339 sys_catalog.cc:458] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: This master's current role is: LEADER
21:26:38.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:38.305675 29599 heartbeater.cc:499] Master 127.28.70.126:45535 was elected leader, sending a full tablet report...
21:26:38.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:38 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067598, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:38.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:38.621213 29480 init.cc:295] Successfully reacquired a new kerberos TGT
21:26:38.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:38.705909 29450 heartbeater.cc:499] Master 127.28.70.126:45535 was elected leader, sending a full tablet report...
21:26:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:38.749323 30109 master_runner.cc:186] Error getting master registration for 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111),
21:26:38.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:38.749897 30109 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:26:39.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:37.989995 30323 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 30299
21:26:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:39.026203 30323 kernel_stack_watchdog.cc:198] Thread 30299 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
21:26:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:26:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:26:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:26:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:26:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:39.026324 30299 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.437s user 0.622s sys 0.934s
21:26:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:39.026662 30299 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.438s user 0.622s sys 0.934s
21:26:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:36.590934 30325 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:39.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:39.029266 30327 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:39.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:39.035323 30326 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2445 milliseconds
21:26:39.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.035346 30299 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:26:39.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.037043 30299 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:26:39.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.047125 29753 heartbeater.cc:499] Master 127.28.70.126:45535 was elected leader, sending a full tablet report...
21:26:39.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.054524 30299 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:26:39.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.056134 30299 hybrid_clock.cc:648] HybridClock initialized: now 1770067599056040 us; error 107 us; skew 500 ppm
21:26:39.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.057356 30299 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:26:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:39 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067599, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.065522 30299 init.cc:377] Logged in from keytab as kudu/127.28.70.124@KRBTEST.COM (short username kudu)
21:26:39.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.071724 30299 webserver.cc:492] Webserver started at http://127.28.70.124:34373/ using document root <none> and password file <none>
21:26:39.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.072714 30299 fs_manager.cc:362] Metadata directory not provided
21:26:39.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.072926 30299 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:26:39.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.081688 30299 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.002s sys 0.005s
21:26:39.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.086956 30361 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:26:39.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.088452 30299 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.000s sys 0.005s
21:26:39.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.088757 30299 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:39.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0da549b46dae45938c88c7a89dae6e27"
21:26:39.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:11 on dist-test-slave-3trw"
21:26:39.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.090790 30299 fs_report.cc:389] FS layout report
21:26:39.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:26:39.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:39.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:39.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data/data
21:26:39.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:26:39.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:26:39.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:26:39.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:26:39.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:26:39.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:26:39.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:26:39.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:26:39.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:26:39.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:26:39.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:26:39.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.156795 30299 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:26:39.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.166261 30299 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:26:39.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.166662 30299 kserver.cc:163] Server-wide thread pool size limit: 3276
21:26:39.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.269896 30299 rpc_server.cc:307] RPC server started. Bound to: 127.28.70.124:42699
21:26:39.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.269985 30413 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.70.124:42699 every 8 connection(s)
21:26:39.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.273129 30299 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data/info.pb
21:26:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.274497 28953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 30299
21:26:39.275 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:39.276 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.70.126:45535(127.28.70.126:45535)] connecting to peer
21:26:39.280 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.28.70.126:45535(127.28.70.126:45535)] Successfully connected to peer
21:26:39.292 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.28.70.126
21:26:39.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.292546 30414 sys_catalog.cc:263] Verifying existing consensus state
21:26:39.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.300804 30414 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: Bootstrap starting.
21:26:39.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:39 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067591, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.70.126@KRBTEST.COM
21:26:39.377 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xf7de0f64, L:/127.0.0.1:56828 - R:/127.28.70.126:45535] using SASL/GSSAPI
21:26:39.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.378034 30414 log.cc:826] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: Log is configured to *not* fsync() on all Append() calls
21:26:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:39 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067598, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.66@KRBTEST.COM for kudu/127.28.70.124@KRBTEST.COM
21:26:39.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.404798 30414 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:26:39.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.405896 30414 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: Bootstrap complete.
21:26:39.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.420997 29597 heartbeater.cc:344] Connected to a master server at 127.28.70.124:42699
21:26:39.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.430356 30414 raft_consensus.cc:359] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:39.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.432523 30414 raft_consensus.cc:740] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0da549b46dae45938c88c7a89dae6e27, State: Initialized, Role: FOLLOWER
21:26:39.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.433176 30414 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:39.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.436084 30420 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:39.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.436874 30420 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: This master's current role is: FOLLOWER
21:26:39.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.437976 30414 sys_catalog.cc:565] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: configured and running, proceeding with master startup.
21:26:39.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.476824 30431 catalog_manager.cc:1269] Loaded cluster ID: 1d6e92849c0c462dbb1161273536d9ce
21:26:39.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.477106 30431 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: loading cluster ID for follower catalog manager: success
21:26:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:39 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067599, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.124@KRBTEST.COM for kudu/127.28.70.126@KRBTEST.COM
21:26:39.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.492862 30431 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: acquiring CA information for follower catalog manager: success
21:26:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.507787 30431 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:26:39.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.563154 30388 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [term 2 FOLLOWER]: Advancing to term 3
21:26:39.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.592725 30420 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 060798801f6146ffa33df46797fa9503. Latest consensus state: current_term: 3 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:39.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.592996 30439 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:39.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.593906 30439 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: This master's current role is: FOLLOWER
21:26:39.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.593997 30420 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: This master's current role is: FOLLOWER
21:26:39.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:39.796067 28658 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9f20 after lost signal to thread 28531
21:26:39.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:39.797163 28658 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9a40 after lost signal to thread 28661
21:26:39.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.841466 29448 heartbeater.cc:344] Connected to a master server at 127.28.70.124:42699
21:26:39.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.848991 30378 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "7de4d9a040ba4467892670bc54c36b8c" instance_seqno: 1770067576174484) as {username='kudu', principal='kudu/127.28.70.65@KRBTEST.COM'} at 127.28.70.65:52411; Asking this server to re-register.
21:26:39.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.850755 29448 heartbeater.cc:461] Registering TS with master...
21:26:39.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:39.854449 30378 ts_manager.cc:194] Registered new tserver with Master: 7de4d9a040ba4467892670bc54c36b8c (127.28.70.65:44823)
21:26:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:40.020037 30355 sys_catalog.cc:455] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "060798801f6146ffa33df46797fa9503" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:40.024189 30355 sys_catalog.cc:458] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: This master's current role is: LEADER
21:26:40.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:40.077944 29748 heartbeater.cc:344] Connected to a master server at 127.28.70.124:42699
21:26:40.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:40.080837 30378 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1091ba5cc3454b1887c6cda9c373ddaa" instance_seqno: 1770067581557030) as {username='kudu', principal='kudu/127.28.70.67@KRBTEST.COM'} at 127.28.70.67:57349; Asking this server to re-register.
21:26:40.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:40.082139 29748 heartbeater.cc:461] Registering TS with master...
21:26:40.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:40.085237 30378 ts_manager.cc:194] Registered new tserver with Master: 1091ba5cc3454b1887c6cda9c373ddaa (127.28.70.67:38533)
21:26:40.384 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:26:40.385 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
21:26:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:40 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067600, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:26:40.430 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:40.430 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc
21:26:40.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:40.427721 30378 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "56b758d0c850420b8576cdd41f6d9658" instance_seqno: 1770067578937003) as {username='kudu', principal='kudu/127.28.70.66@KRBTEST.COM'} at 127.28.70.66:47547; Asking this server to re-register.
21:26:40.432 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:26:40.432 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.70.126:45535
21:26:40.432 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
21:26:40.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:40.433228 29597 heartbeater.cc:461] Registering TS with master...
21:26:40.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:40.434544 28953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 30109
21:26:40.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:40.437067 30378 ts_manager.cc:194] Registered new tserver with Master: 56b758d0c850420b8576cdd41f6d9658 (127.28.70.66:40871)
21:26:40.479 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.28.70.126:45535(127.28.70.126:45535)] handling channelInactive
21:26:40.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.70.126:45535(127.28.70.126:45535)] cleaning up while in state READY due to: connection closed
21:26:40.484 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:40.484 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.70.125:34925
21:26:40.484 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
21:26:40.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:40.484716 28953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 30190
21:26:40.529 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:40.529 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.70.124:42699
21:26:40.530 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:26:40.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:40.530500 28953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 30299
21:26:40.560 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:40.560 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.70.126:45535
21:26:40.561 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
21:26:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:40.562317 28953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data
21:26:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:26:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:26:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:26:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/logs
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data/info.pb
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.126:45535
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.126
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46269
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.126
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:26:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:26:40.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:40.912649 30448 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:26:40.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:40.913318 30448 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:26:40.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:40.914130 30448 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:26:40.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:40.953533 30448 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:26:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:40.953873 30448 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:26:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:40.954358 30448 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:26:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:40.954704 30448 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:26:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:40.955024 30448 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:26:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:40.955349 30448 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:26:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:40.955590 30448 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45551/jwks.json
21:26:41.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:41.004391 30448 master_runner.cc:386] Master server non-default flags:
21:26:41.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.126
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.126:45535
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:41.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data/info.pb
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.126
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46269
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/logs
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:26:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:26:41.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:26:41.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:26:41.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:41.005767 30448 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:26:41.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:41.007010 30448 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:26:41.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:41.021139 30457 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:41.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:41 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067601, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:41.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:41.317598 29631 init.cc:295] Successfully reacquired a new kerberos TGT
21:26:42.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:42.421617 30455 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 30448
21:26:42.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:41.022456 30456 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:42.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:42.495993 30448 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.474s user 0.499s sys 0.952s
21:26:42.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:42.496842 30448 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.475s user 0.499s sys 0.952s
21:26:42.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:42.497342 30458 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
21:26:42.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.498778 30448 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:26:42.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:42.498879 30459 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:42.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.502626 30448 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:26:42.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.505285 30448 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:26:42.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.506672 30448 hybrid_clock.cc:648] HybridClock initialized: now 1770067602506641 us; error 37 us; skew 500 ppm
21:26:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.507556 30448 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:26:42.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:42 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067602, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.514096 30448 init.cc:377] Logged in from keytab as kudu/127.28.70.126@KRBTEST.COM (short username kudu)
21:26:42.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.518695 30448 webserver.cc:492] Webserver started at http://127.28.70.126:46269/ using document root <none> and password file <none>
21:26:42.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.519771 30448 fs_manager.cc:362] Metadata directory not provided
21:26:42.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.519989 30448 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:26:42.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.527765 30448 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.004s sys 0.000s
21:26:42.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.532114 30474 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:26:42.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.533164 30448 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.000s sys 0.004s
21:26:42.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.533476 30448 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:42.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "060798801f6146ffa33df46797fa9503"
21:26:42.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:06 on dist-test-slave-3trw"
21:26:42.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.535418 30448 fs_report.cc:389] FS layout report
21:26:42.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:26:42.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:42.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/wal
21:26:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data/data
21:26:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:26:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:26:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:26:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:26:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:26:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:26:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:26:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:26:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:26:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:26:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:26:42.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.597295 30448 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:26:42.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.602864 30448 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:26:42.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.603278 30448 kserver.cc:163] Server-wide thread pool size limit: 3276
21:26:42.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.678861 30448 rpc_server.cc:307] RPC server started. Bound to: 127.28.70.126:45535
21:26:42.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.678913 30525 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.70.126:45535 every 8 connection(s)
21:26:42.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.681659 30448 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-0/data/info.pb
21:26:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.682746 28953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 30448
21:26:42.683 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:42.683 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.70.125:34925
21:26:42.684 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
21:26:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.686123 28953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data
21:26:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:26:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:26:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:26:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/logs
21:26:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data/info.pb
21:26:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.125:34925
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.125
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38211
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.125
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:42.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:42.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:26:42.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:26:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.696115 30526 sys_catalog.cc:263] Verifying existing consensus state
21:26:42.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.705411 30526 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Bootstrap starting.
21:26:42.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.760119 30526 log.cc:826] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Log is configured to *not* fsync() on all Append() calls
21:26:42.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.781762 30526 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:26:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.782781 30526 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: Bootstrap complete.
21:26:42.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.820420 30526 raft_consensus.cc:359] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:42.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.824362 30526 raft_consensus.cc:740] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 060798801f6146ffa33df46797fa9503, State: Initialized, Role: FOLLOWER
21:26:42.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.825523 30526 consensus_queue.cc:260] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.847162 30532 sys_catalog.cc:455] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.847981 30532 sys_catalog.cc:458] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: This master's current role is: FOLLOWER
21:26:42.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.851917 30526 sys_catalog.cc:565] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: configured and running, proceeding with master startup.
21:26:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:42 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067595, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.65@KRBTEST.COM for kudu/127.28.70.126@KRBTEST.COM
21:26:42.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.885994 29450 heartbeater.cc:344] Connected to a master server at 127.28.70.126:45535
21:26:42.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:42 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067602, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.126@KRBTEST.COM for kudu/127.28.70.126@KRBTEST.COM
21:26:42.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.918637 30545 catalog_manager.cc:1269] Loaded cluster ID: 1d6e92849c0c462dbb1161273536d9ce
21:26:42.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.926826 30545 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: loading cluster ID for follower catalog manager: success
21:26:42.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.934024 30545 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: acquiring CA information for follower catalog manager: success
21:26:42.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.939106 30545 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:26:42.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.943578 30448 master_runner.cc:186] Error getting master registration for 127.28.70.125:34925: Network error: Client connection negotiation failed: client connection to 127.28.70.125:34925: connect: Connection refused (error 111),
21:26:42.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:42.989648 30448 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:26:43.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:43.087934 29753 heartbeater.cc:344] Connected to a master server at 127.28.70.126:45535
21:26:43.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:43.092299 30491 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1091ba5cc3454b1887c6cda9c373ddaa" instance_seqno: 1770067581557030) as {username='kudu', principal='kudu/127.28.70.67@KRBTEST.COM'} at 127.28.70.67:54713; Asking this server to re-register.
21:26:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:43.093544 29753 heartbeater.cc:461] Registering TS with master...
21:26:43.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:43.097455 30491 ts_manager.cc:194] Registered new tserver with Master: 1091ba5cc3454b1887c6cda9c373ddaa (127.28.70.67:38533)
21:26:43.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:43.134258 30528 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:26:43.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:43.134784 30528 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:26:43.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:43.135396 30528 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:26:43.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:43.169064 30528 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:26:43.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:43.169492 30528 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:26:43.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:43.169960 30528 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:26:43.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:43.170190 30528 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:26:43.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:43.170390 30528 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:26:43.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:43.170750 30528 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:26:43.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:43.170917 30528 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45551/jwks.json
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:43.210609 30528 master_runner.cc:386] Master server non-default flags:
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.125
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.125:34925
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data/info.pb
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.125
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38211
21:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:26:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:26:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:26:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/logs
21:26:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:26:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:26:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:26:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:26:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:26:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:26:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:26:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:26:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:26:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:43.212029 30528 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:26:43.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:43.213058 30528 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:26:43.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:43.226053 30554 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:43.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:43.391681 29599 heartbeater.cc:344] Connected to a master server at 127.28.70.126:45535
21:26:43.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:43.396440 30491 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "56b758d0c850420b8576cdd41f6d9658" instance_seqno: 1770067578937003) as {username='kudu', principal='kudu/127.28.70.66@KRBTEST.COM'} at 127.28.70.66:57397; Asking this server to re-register.
21:26:43.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:43.398324 29599 heartbeater.cc:461] Registering TS with master...
21:26:43.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:43.402454 30491 ts_manager.cc:194] Registered new tserver with Master: 56b758d0c850420b8576cdd41f6d9658 (127.28.70.66:40871)
21:26:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:43.894163 30491 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "7de4d9a040ba4467892670bc54c36b8c" instance_seqno: 1770067576174484) as {username='kudu', principal='kudu/127.28.70.65@KRBTEST.COM'} at 127.28.70.65:38141; Asking this server to re-register.
21:26:43.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:43.896500 29450 heartbeater.cc:461] Registering TS with master...
21:26:43.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:43.899869 30491 ts_manager.cc:194] Registered new tserver with Master: 7de4d9a040ba4467892670bc54c36b8c (127.28.70.65:44823)
21:26:44.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:44.158607 30564 raft_consensus.cc:493] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:26:44.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:44.159590 30564 raft_consensus.cc:515] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:44.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:44.188283 30564 leader_election.cc:290] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers c07e7962716a40a8bb9a84004a578cfb (127.28.70.125:34925), 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699)
21:26:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:44.199566 30477 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:44.210425 30476 leader_election.cc:336] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c07e7962716a40a8bb9a84004a578cfb (127.28.70.125:34925): Network error: Client connection negotiation failed: client connection to 127.28.70.125:34925: connect: Connection refused (error 111)
21:26:44.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:44.215178 30477 leader_election.cc:336] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699): Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:44.215740 30477 leader_election.cc:304] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 060798801f6146ffa33df46797fa9503; no voters: 0da549b46dae45938c88c7a89dae6e27, c07e7962716a40a8bb9a84004a578cfb
21:26:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:44.216710 30564 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:26:44.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:44.831959 29744 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9240 after lost signal to thread 29614
21:26:44.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:44.833652 29744 debug-util.cc:398] Leaking SignalData structure 0x7b08000b42a0 after lost signal to thread 29747
21:26:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:45 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067605, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:45.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:45.467825 29329 init.cc:295] Successfully reacquired a new kerberos TGT
21:26:45.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:45.853542 30528 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.626s user 0.757s sys 1.464s
21:26:45.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:45.855861 30528 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.628s user 0.757s sys 1.464s
21:26:45.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:45.856168 30528 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:26:45.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:44.627786 30553 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 30528
21:26:45.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:45.855079 30557 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:45.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:45.855262 30556 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2626 milliseconds
21:26:45.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:43.229346 30555 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:45.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:45.863112 30528 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:26:45.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:45.874295 30528 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:26:45.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:45.875768 30528 hybrid_clock.cc:648] HybridClock initialized: now 1770067605875723 us; error 56 us; skew 500 ppm
21:26:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:45.876999 30528 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:26:45.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:45 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067605, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:45.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:45.885994 30528 init.cc:377] Logged in from keytab as kudu/127.28.70.125@KRBTEST.COM (short username kudu)
21:26:45.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:45.900391 30528 webserver.cc:492] Webserver started at http://127.28.70.125:38211/ using document root <none> and password file <none>
21:26:45.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:45.902573 30528 fs_manager.cc:362] Metadata directory not provided
21:26:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:45.902875 30528 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:26:45.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:45.913988 30528 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.001s
21:26:45.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:45.920071 30579 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:26:45.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:45.921383 30528 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
21:26:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:45.921887 30528 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c07e7962716a40a8bb9a84004a578cfb"
21:26:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:08 on dist-test-slave-3trw"
21:26:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:45.924641 30528 fs_report.cc:389] FS layout report
21:26:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:26:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/wal
21:26:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data/data
21:26:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:26:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:26:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:26:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:26:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:26:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:26:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:26:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:26:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:26:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:26:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:26:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:46.039500 28305 debug-util.cc:398] Leaking SignalData structure 0x7b08000909e0 after lost signal to thread 28242
21:26:46.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:46.040274 28305 debug-util.cc:398] Leaking SignalData structure 0x7b0800074000 after lost signal to thread 28308
21:26:46.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.069072 30528 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:26:46.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.085796 30528 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:26:46.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.086473 30528 kserver.cc:163] Server-wide thread pool size limit: 3276
21:26:46.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.271062 30600 raft_consensus.cc:493] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:26:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.271602 30600 raft_consensus.cc:515] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:46.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.303617 30600 leader_election.cc:290] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers c07e7962716a40a8bb9a84004a578cfb (127.28.70.125:34925), 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699)
21:26:46.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:46.311847 30477 leader_election.cc:336] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699): Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:46.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:46.314929 30476 leader_election.cc:336] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c07e7962716a40a8bb9a84004a578cfb (127.28.70.125:34925): Network error: Client connection negotiation failed: client connection to 127.28.70.125:34925: connect: Connection refused (error 111)
21:26:46.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.315389 30476 leader_election.cc:304] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 060798801f6146ffa33df46797fa9503; no voters: 0da549b46dae45938c88c7a89dae6e27, c07e7962716a40a8bb9a84004a578cfb
21:26:46.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.316170 30600 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:26:46.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.559029 30636 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.70.125:34925 every 8 connection(s)
21:26:46.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.559376 30528 rpc_server.cc:307] RPC server started. Bound to: 127.28.70.125:34925
21:26:46.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.566097 30528 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-1/data/info.pb
21:26:46.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.574519 28953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 30528
21:26:46.575 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:46.575 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.70.124:42699
21:26:46.575 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
21:26:46.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.577584 28953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:46.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu
21:26:46.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:46.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/logs
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data/info.pb
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.124:42699
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.124
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34373
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.124
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:46.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:46.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:46.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:46.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:46.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:46.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:46.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:26:46.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:26:46.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.597437 30637 sys_catalog.cc:263] Verifying existing consensus state
21:26:46.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.606951 30637 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: Bootstrap starting.
21:26:46.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.705474 30637 log.cc:826] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: Log is configured to *not* fsync() on all Append() calls
21:26:46.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.738655 30637 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:26:46.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.739917 30637 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: Bootstrap complete.
21:26:46.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.780836 30637 raft_consensus.cc:359] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:46.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.784829 30637 raft_consensus.cc:740] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: c07e7962716a40a8bb9a84004a578cfb, State: Initialized, Role: FOLLOWER
21:26:46.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.786058 30637 consensus_queue.cc:260] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.806499 30641 sys_catalog.cc:455] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.807390 30641 sys_catalog.cc:458] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: This master's current role is: FOLLOWER
21:26:46.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.809648 30637 sys_catalog.cc:565] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: configured and running, proceeding with master startup.
21:26:46.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:46 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067605, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.65@KRBTEST.COM for kudu/127.28.70.125@KRBTEST.COM
21:26:46.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.922264 30655 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:26:46.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.957123 29449 heartbeater.cc:344] Connected to a master server at 127.28.70.125:34925
21:26:46.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.964483 30597 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "7de4d9a040ba4467892670bc54c36b8c" instance_seqno: 1770067576174484) as {username='kudu', principal='kudu/127.28.70.65@KRBTEST.COM'} at 127.28.70.65:54537; Asking this server to re-register.
21:26:46.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.966519 29449 heartbeater.cc:461] Registering TS with master...
21:26:46.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:46.971529 30596 ts_manager.cc:194] Registered new tserver with Master: 7de4d9a040ba4467892670bc54c36b8c (127.28.70.65:44823)
21:26:47.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:47 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067605, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.125@KRBTEST.COM for kudu/127.28.70.126@KRBTEST.COM
21:26:47.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:47 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067605, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.125@KRBTEST.COM for kudu/127.28.70.125@KRBTEST.COM
21:26:47.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:47.091909 30528 master_runner.cc:186] Error getting master registration for 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111),
21:26:47.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:47.132470 30528 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:26:47.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:47 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067601, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.67@KRBTEST.COM for kudu/127.28.70.125@KRBTEST.COM
21:26:47.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:47.149734 29749 heartbeater.cc:344] Connected to a master server at 127.28.70.125:34925
21:26:47.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:47.152382 30596 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1091ba5cc3454b1887c6cda9c373ddaa" instance_seqno: 1770067581557030) as {username='kudu', principal='kudu/127.28.70.67@KRBTEST.COM'} at 127.28.70.67:34987; Asking this server to re-register.
21:26:47.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:47.153880 29749 heartbeater.cc:461] Registering TS with master...
21:26:47.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:47.156930 30596 ts_manager.cc:194] Registered new tserver with Master: 1091ba5cc3454b1887c6cda9c373ddaa (127.28.70.67:38533)
21:26:47.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:47.348654 30639 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:26:47.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:47.349460 30639 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:26:47.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:47.350309 30639 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:26:47.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:47.386523 30639 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:26:47.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:47.386801 30639 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:26:47.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:47.387245 30639 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:26:47.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:47.387614 30639 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:26:47.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:47.387940 30639 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:26:47.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:47.388267 30639 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:26:47.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:47.388521 30639 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45551/jwks.json
21:26:47.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:47.430845 30639 master_runner.cc:386] Master server non-default flags:
21:26:47.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:26:47.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.70.84:32833
21:26:47.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:26:47.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:26:47.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data
21:26:47.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:47.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:26:47.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:26:47.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.70.126:45535,127.28.70.125:34925,127.28.70.124:42699
21:26:47.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/krb5kdc/kudu.keytab
21:26:47.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.70.124
21:26:47.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:26:47.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:26:47.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.70.124:42699
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45551/jwks.json
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data/info.pb
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.70.124
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34373
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/logs
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6dc40cf777cf7c7ac02d2c323748b12f8c79a143
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 20:56:03 UTC on 5fd53c4cbb9d
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10317
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:47.432474 30639 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:26:47.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:47.433811 30639 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:26:47.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:47 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067598, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.66@KRBTEST.COM for kudu/127.28.70.125@KRBTEST.COM
21:26:47.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:47.474689 29598 heartbeater.cc:344] Connected to a master server at 127.28.70.125:34925
21:26:47.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:47.476728 30669 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:47.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:47.477710 30596 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "56b758d0c850420b8576cdd41f6d9658" instance_seqno: 1770067578937003) as {username='kudu', principal='kudu/127.28.70.66@KRBTEST.COM'} at 127.28.70.66:55287; Asking this server to re-register.
21:26:47.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:47.479527 29598 heartbeater.cc:461] Registering TS with master...
21:26:47.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:47.482707 30596 ts_manager.cc:194] Registered new tserver with Master: 56b758d0c850420b8576cdd41f6d9658 (127.28.70.66:40871)
21:26:47.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:47.970996 30655 catalog_manager.cc:1269] Loaded cluster ID: 1d6e92849c0c462dbb1161273536d9ce
21:26:47.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:47.971347 30655 catalog_manager.cc:1562] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: loading cluster ID for follower catalog manager: success
21:26:48.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:48 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067602, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.126@KRBTEST.COM for kudu/127.28.70.125@KRBTEST.COM
21:26:48.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.119503 30655 catalog_manager.cc:1584] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: acquiring CA information for follower catalog manager: success
21:26:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.183327 30448 master_runner.cc:186] Error getting master registration for 127.28.70.124:42699: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111),
21:26:48.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.183903 30448 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:26:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.184973 30655 catalog_manager.cc:1612] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:26:48.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.273805 30681 raft_consensus.cc:493] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:26:48.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.274518 30681 raft_consensus.cc:515] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:48.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.278985 30501 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c07e7962716a40a8bb9a84004a578cfb" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "060798801f6146ffa33df46797fa9503" is_pre_election: true
21:26:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.279857 30501 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c07e7962716a40a8bb9a84004a578cfb in term 3.
21:26:48.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.281540 30582 leader_election.cc:304] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 060798801f6146ffa33df46797fa9503, c07e7962716a40a8bb9a84004a578cfb; no voters:
21:26:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.300493 30681 leader_election.cc:290] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 060798801f6146ffa33df46797fa9503 (127.28.70.126:45535), 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699)
21:26:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.301123 30681 raft_consensus.cc:2804] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 3 FOLLOWER]: Leader pre-election won for term 4
21:26:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.301553 30681 raft_consensus.cc:493] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:26:48.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.301954 30681 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 3 FOLLOWER]: Advancing to term 4
21:26:48.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:48.312525 30582 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:48.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:48.328634 30582 leader_election.cc:336] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699): Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:48.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.351397 30681 raft_consensus.cc:515] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:48.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.354763 30501 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c07e7962716a40a8bb9a84004a578cfb" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "060798801f6146ffa33df46797fa9503"
21:26:48.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.355408 30501 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 3 FOLLOWER]: Advancing to term 4
21:26:48.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.371750 30501 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c07e7962716a40a8bb9a84004a578cfb in term 4.
21:26:48.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:48.371698 30582 leader_election.cc:336] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699): Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111)
21:26:48.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.372382 30681 leader_election.cc:290] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [CANDIDATE]: Term 4 election: Requested vote from peers 060798801f6146ffa33df46797fa9503 (127.28.70.126:45535), 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699)
21:26:48.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.375485 30582 leader_election.cc:304] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 060798801f6146ffa33df46797fa9503, c07e7962716a40a8bb9a84004a578cfb; no voters: 0da549b46dae45938c88c7a89dae6e27
21:26:48.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.376542 30682 raft_consensus.cc:2804] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 4 FOLLOWER]: Leader election won for term 4
21:26:48.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.395839 30682 raft_consensus.cc:697] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [term 4 LEADER]: Becoming Leader. State: Replica: c07e7962716a40a8bb9a84004a578cfb, State: Running, Role: LEADER
21:26:48.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.397183 30682 consensus_queue.cc:237] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.421489 30681 sys_catalog.cc:455] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: SysCatalogTable state changed. Reason: New leader c07e7962716a40a8bb9a84004a578cfb. Latest consensus state: current_term: 4 leader_uuid: "c07e7962716a40a8bb9a84004a578cfb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:48.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.422494 30681 sys_catalog.cc:458] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: This master's current role is: LEADER
21:26:48.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.426970 30687 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:26:48.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.466362 30687 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:26:48.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.475847 30687 catalog_manager.cc:1269] Loaded cluster ID: 1d6e92849c0c462dbb1161273536d9ce
21:26:48.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.476179 30687 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:26:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.507511 30687 catalog_manager.cc:1514] Loading token signing keys...
21:26:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.513136 30687 catalog_manager.cc:6038] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb: Loaded TSK: 0
21:26:48.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.526933 30687 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:26:48.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:48 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067608, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:48.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.625145 29480 init.cc:295] Successfully reacquired a new kerberos TGT
21:26:48.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:48.810585 30582 consensus_peers.cc:597] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb -> Peer 0da549b46dae45938c88c7a89dae6e27 (127.28.70.124:42699): Couldn't send request to peer 0da549b46dae45938c88c7a89dae6e27. Status: Network error: Client connection negotiation failed: client connection to 127.28.70.124:42699: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:26:48.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.839306 30501 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [term 4 FOLLOWER]: Refusing update from remote peer c07e7962716a40a8bb9a84004a578cfb: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
21:26:48.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.841384 30681 consensus_queue.cc:1048] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [LEADER]: Connected to new peer: Peer: permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.001s
21:26:48.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.878602 30689 sys_catalog.cc:455] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c07e7962716a40a8bb9a84004a578cfb. Latest consensus state: current_term: 4 leader_uuid: "c07e7962716a40a8bb9a84004a578cfb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:48.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.881150 30689 sys_catalog.cc:458] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: This master's current role is: FOLLOWER
21:26:48.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.905664 30685 sys_catalog.cc:455] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "c07e7962716a40a8bb9a84004a578cfb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:48.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.906662 30685 sys_catalog.cc:458] T 00000000000000000000000000000000 P c07e7962716a40a8bb9a84004a578cfb [sys.catalog]: This master's current role is: LEADER
21:26:48.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.912436 30689 sys_catalog.cc:455] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "c07e7962716a40a8bb9a84004a578cfb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:48.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:48.913122 30689 sys_catalog.cc:458] T 00000000000000000000000000000000 P 060798801f6146ffa33df46797fa9503 [sys.catalog]: This master's current role is: FOLLOWER
21:26:49.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:48.857609 30667 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 30639
21:26:49.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:49.657861 30639 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.173s user 0.731s sys 1.217s
21:26:49.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:49.658929 30639 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.174s user 0.732s sys 1.217s
21:26:49.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:47.487402 30671 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:49.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:49.682377 30701 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:26:49.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:49.682652 30639 server_base.cc:1060] running on GCE node
21:26:49.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:49.684122 30639 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:26:49.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:49.687199 30639 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:26:49.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:49.688611 30639 hybrid_clock.cc:648] HybridClock initialized: now 1770067609688573 us; error 38 us; skew 500 ppm
21:26:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:49.689486 30639 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:26:49.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:49 dist-test-slave-3trw krb5kdc[28965](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067609, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:26:49.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:49.697045 30639 init.cc:377] Logged in from keytab as kudu/127.28.70.124@KRBTEST.COM (short username kudu)
21:26:49.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:49.702527 30639 webserver.cc:492] Webserver started at http://127.28.70.124:34373/ using document root <none> and password file <none>
21:26:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:49.703469 30522 debug-util.cc:398] Leaking SignalData structure 0x7b0800071260 after lost signal to thread 30450
21:26:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:49.703733 30639 fs_manager.cc:362] Metadata directory not provided
21:26:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:49.704005 30639 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:26:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:49.704028 30522 debug-util.cc:398] Leaking SignalData structure 0x7b0800088d20 after lost signal to thread 30525
21:26:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:49.732142 30639 fs_manager.cc:714] Time spent opening directory manager: real 0.024s user 0.011s sys 0.011s
21:26:49.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:49.738739 30707 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:26:49.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:49.740314 30639 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.000s sys 0.004s
21:26:49.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:49.740736 30639 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data,/tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:49.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0da549b46dae45938c88c7a89dae6e27"
21:26:49.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 21:26:11 on dist-test-slave-3trw"
21:26:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:49.743695 30639 fs_report.cc:389] FS layout report
21:26:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:26:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/wal
21:26:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data/data
21:26:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:26:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:26:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:26:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:26:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:26:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:26:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:26:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:26:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:26:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:26:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:26:49.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:49.820616 30639 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:26:49.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:49.828326 30639 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:26:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:49.828903 30639 kserver.cc:163] Server-wide thread pool size limit: 3276
21:26:50.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.036660 29449 heartbeater.cc:499] Master 127.28.70.125:34925 was elected leader, sending a full tablet report...
21:26:50.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.167131 29749 heartbeater.cc:499] Master 127.28.70.125:34925 was elected leader, sending a full tablet report...
21:26:50.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.214632 30639 rpc_server.cc:307] RPC server started. Bound to: 127.28.70.124:42699
21:26:50.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.214704 30765 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.70.124:42699 every 8 connection(s)
21:26:50.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.218142 30639 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskurmOAe/test-tmp/mini-kudu-cluster7059956738535092572/master-2/data/info.pb
21:26:50.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.227017 28953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu as pid 30639
21:26:50.227 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:26:50.228 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.70.126:45535(127.28.70.126:45535)] connecting to peer
21:26:50.231 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.28.70.126:45535(127.28.70.126:45535)] Successfully connected to peer
21:26:50.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.238387 30766 sys_catalog.cc:263] Verifying existing consensus state
21:26:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.247072 30766 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: Bootstrap starting.
21:26:50.251 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.70.126:45535(127.28.70.126:45535)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
21:26:50.252 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.28.70.126:45535(127.28.70.126:45535)] handling channelInactive
21:26:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 21:26:50.255319 30768 negotiation.cc:337] Failed RPC negotiation. Trace:
21:26:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 21:26:50.231071 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:42988 (local address 127.28.70.126:45535)
21:26:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 21:26:50.248990 (+ 17919us) server_negotiation.cc:207] Beginning negotiation
21:26:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 21:26:50.249019 (+ 29us) server_negotiation.cc:400] Waiting for connection header
21:26:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 21:26:50.249072 (+ 53us) server_negotiation.cc:408] Connection header received
21:26:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 21:26:50.249444 (+ 372us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:26:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 21:26:50.249476 (+ 32us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:26:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 21:26:50.249737 (+ 261us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:26:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 21:26:50.252193 (+ 2456us) server_negotiation.cc:227] Negotiated authn=SASL
21:26:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 21:26:50.253931 (+ 1738us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:42988: BlockingRecv error: recv got EOF from 127.0.0.1:42988 (error 108)
21:26:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":16924,"thread_start_us":17376,"threads_started":1}
21:26:50.258 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.70.126:45535(127.28.70.126:45535)] disconnecting while in state TERMINATED
21:26:50.261 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.70.125:34925(127.28.70.125:34925)] disconnecting while in state TERMINATED
21:26:50.262 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.70.124:42699(127.28.70.124:42699)] disconnecting while in state TERMINATED
21:26:50.274 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:26:50.285 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:26:50.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.307142 28953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 29280
21:26:50.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.326133 30766 log.cc:826] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: Log is configured to *not* fsync() on all Append() calls
21:26:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.379086 28953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 29462
21:26:50.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 21:26:50 dist-test-slave-3trw krb5kdc[28965](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770067605, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.70.125@KRBTEST.COM for kudu/127.28.70.124@KRBTEST.COM
21:26:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.407420 30766 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:26:50.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.408497 30766 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: Bootstrap complete.
21:26:50.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.441442 30766 raft_consensus.cc:359] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:50.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.446821 28953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 29611
21:26:50.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.445708 30766 raft_consensus.cc:740] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0da549b46dae45938c88c7a89dae6e27, State: Initialized, Role: FOLLOWER
21:26:50.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.458216 30766 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } }
21:26:50.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.485394 28953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 30448
21:26:50.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.481177 30776 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:50.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.488197 30776 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: This master's current role is: FOLLOWER
21:26:50.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.491904 30740 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [term 3 FOLLOWER]: Advancing to term 4
21:26:50.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.495395 30766 sys_catalog.cc:565] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: configured and running, proceeding with master startup.
21:26:50.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.535979 30776 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c07e7962716a40a8bb9a84004a578cfb. Latest consensus state: current_term: 4 leader_uuid: "c07e7962716a40a8bb9a84004a578cfb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:50.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.536723 30776 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: This master's current role is: FOLLOWER
21:26:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.537742 30782 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "c07e7962716a40a8bb9a84004a578cfb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "060798801f6146ffa33df46797fa9503" member_type: VOTER last_known_addr { host: "127.28.70.126" port: 45535 } } peers { permanent_uuid: "c07e7962716a40a8bb9a84004a578cfb" member_type: VOTER last_known_addr { host: "127.28.70.125" port: 34925 } } peers { permanent_uuid: "0da549b46dae45938c88c7a89dae6e27" member_type: VOTER last_known_addr { host: "127.28.70.124" port: 42699 } } }
21:26:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.538473 30782 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27 [sys.catalog]: This master's current role is: FOLLOWER
21:26:50.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.557996 28953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 30528
21:26:50.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.582955 30790 catalog_manager.cc:1269] Loaded cluster ID: 1d6e92849c0c462dbb1161273536d9ce
21:26:50.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.583235 30790 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: loading cluster ID for follower catalog manager: success
21:26:50.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.590276 30790 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: acquiring CA information for follower catalog manager: success
21:26:50.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.595985 30790 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 0da549b46dae45938c88c7a89dae6e27: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:26:50.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.573599 30639 master_runner.cc:186] Error getting master registration for 127.28.70.126:45535: Network error: Client connection negotiation failed: client connection to 127.28.70.126:45535: connect: Connection refused (error 111),
21:26:50.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.599862 30639 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:26:50.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:50.603358 28953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 30639
21:26:51.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-02-02T21:26:51Z chronyd exiting
21:26:52.855 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:26:52.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:52.856666 28229 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 28515
21:26:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:52.883766 28229 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 28672
21:26:52.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:52.911549 28229 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 28812
21:26:52.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:52.940882 28229 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 28241
21:26:52.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:52.978437 28229 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 28311
21:26:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 21:26:53.009229 28229 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskurmOAe/build/tsan/bin/kudu with pid 28386
21:26:54.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-02-02T21:26:54Z chronyd exiting
21:26:55.189 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more