03:08:08.156 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:08:08.156 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task67Ldfr/test-tmp
03:08:08.157 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:08:08.157 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task67Ldfr/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:08:08.160 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:08:08.525 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:08:08.525 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:08:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:08.528764 15628 mini_oidc.cc:150] Starting JWKS server
03:08:08.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:08.532274 15628 webserver.cc:492] Webserver started at http://127.0.0.1:44009/ using document root <none> and password file <none>
03:08:08.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:08.532716 15628 mini_oidc.cc:186] Starting OIDC Discovery server
03:08:08.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:08.534575 15628 webserver.cc:492] Webserver started at http://127.0.0.1:44923/ using document root <none> and password file <none>
03:08:08.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-21T03:08:08Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
03:08:08.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-21T03:08:08Z Disabled control of system clock
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:08.602458 15628 external_mini_cluster.cc:1366] Running /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-0/wal
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-0/data
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-0/logs
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-0/data/info.pb
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.67.62:38603
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.67.62
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.67.20:40279
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.67.62:38603,127.15.67.61:40499,127.15.67.60:46163
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44009/jwks.json with env {}
03:08:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:08.924510 15640 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:08.925107 15640 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:08.925601 15640 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:08.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:08.960052 15640 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:08.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:08.960449 15640 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:08.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:08.960672 15640 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:08.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:08.960875 15640 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:08.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:08.961076 15640 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:08.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:08.961225 15640 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44009/jwks.json
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:09.001843 15640 master_runner.cc:386] Master server non-default flags:
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.67.20:40279
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-0/data
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-0/wal
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.67.62:38603,127.15.67.61:40499,127.15.67.60:46163
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.67.62:38603
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44009/jwks.json
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-0/data/info.pb
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.67.62
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-0/logs
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 68f10dd41067059260a5fb5c725b178b69c38aed
03:08:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:09.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Mar 2026 02:43:22 UTC on e7f111948823
03:08:09.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11031
03:08:09.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:09.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:09.003396 15640 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:09.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:09.006486 15640 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:09.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:09.025197 15647 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:10.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:09.025580 15646 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:10.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.329967 15649 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:10.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.335578 15640 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.313s user 0.472s sys 0.828s
03:08:10.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.335651 15648 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1304 milliseconds
03:08:10.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.335940 15640 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.313s user 0.472s sys 0.828s
03:08:10.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.336264 15640 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.337541 15640 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:10.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.340438 15640 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.342203 15640 hybrid_clock.cc:648] HybridClock initialized: now 1774062490342120 us; error 98 us; skew 500 ppm
03:08:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.343225 15640 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:10.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.351936 15640 webserver.cc:492] Webserver started at http://127.15.67.62:33889/ using document root <none> and password file <none>
03:08:10.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.353042 15640 fs_manager.cc:362] Metadata directory not provided
03:08:10.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.353341 15640 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.353881 15640 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:08:10.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.358764 15640 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-0/data/instance:
03:08:10.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "453ece4e574d4392bc70917f43a42108"
03:08:10.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:10 on dist-test-slave-62xr"
03:08:10.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.360136 15640 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-0/wal/instance:
03:08:10.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "453ece4e574d4392bc70917f43a42108"
03:08:10.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:10 on dist-test-slave-62xr"
03:08:10.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.370838 15640 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.001s sys 0.009s
03:08:10.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.378404 15656 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.379913 15640 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.002s
03:08:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.380455 15640 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-0/data,/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-0/wal
03:08:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "453ece4e574d4392bc70917f43a42108"
03:08:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:10 on dist-test-slave-62xr"
03:08:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.380848 15640 fs_report.cc:389] FS layout report
03:08:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-0/wal
03:08:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-0/wal
03:08:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-0/data/data
03:08:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.472908 15640 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:10.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.474704 15640 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:10.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.475414 15640 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.556429 15640 rpc_server.cc:307] RPC server started. Bound to: 127.15.67.62:38603
03:08:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.556463 15707 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.67.62:38603 every 8 connection(s)
03:08:10.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.559463 15640 server_base.cc:1193] Dumped server information to /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-0/data/info.pb
03:08:10.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.565793 15708 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:08:10.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.567320 15628 external_mini_cluster.cc:1428] Started /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu as pid 15640
03:08:10.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.567924 15628 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-0/wal/instance
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.571794 15628 external_mini_cluster.cc:1366] Running /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-1/wal
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-1/data
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-1/logs
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-1/data/info.pb
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.67.61:40499
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.67.61
03:08:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:08:10.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.67.20:40279
03:08:10.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:10.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:10.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:10.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:10.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.67.62:38603,127.15.67.61:40499,127.15.67.60:46163
03:08:10.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:10.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44009/jwks.json with env {}
03:08:10.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.578433 15708 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } has no permanent_uuid. Determining permanent_uuid...
03:08:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.612617 15708 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } has no permanent_uuid. Determining permanent_uuid...
03:08:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.615578 15660 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.67.61:40499: connect: Connection refused (error 111)
03:08:10.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.620524 15708 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.61:40499: Network error: Client connection negotiation failed: client connection to 127.15.67.61:40499: connect: Connection refused (error 111)
03:08:10.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.670279 15708 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } attempt: 1
03:08:10.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.674831 15708 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.61:40499: Network error: Client connection negotiation failed: client connection to 127.15.67.61:40499: connect: Connection refused (error 111)
03:08:10.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.743453 15708 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } attempt: 2
03:08:10.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.748282 15708 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.61:40499: Network error: Client connection negotiation failed: client connection to 127.15.67.61:40499: connect: Connection refused (error 111)
03:08:10.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.839787 15708 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } attempt: 3
03:08:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.844050 15708 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.61:40499: Network error: Client connection negotiation failed: client connection to 127.15.67.61:40499: connect: Connection refused (error 111)
03:08:10.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.906525 15710 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:10.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.907083 15710 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:10.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.907689 15710 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:10.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.941233 15710 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:10.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.941543 15710 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:10.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.941738 15710 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:10.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.941941 15710 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:10.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.942138 15710 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:10.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.942294 15710 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44009/jwks.json
03:08:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.980640 15710 master_runner.cc:386] Master server non-default flags:
03:08:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.67.20:40279
03:08:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-1/data
03:08:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-1/wal
03:08:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.67.62:38603,127.15.67.61:40499,127.15.67.60:46163
03:08:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.67.61:40499
03:08:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44009/jwks.json
03:08:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-1/data/info.pb
03:08:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.67.61
03:08:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:08:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-1/logs
03:08:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 68f10dd41067059260a5fb5c725b178b69c38aed
03:08:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Mar 2026 02:43:22 UTC on e7f111948823
03:08:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11031
03:08:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.982154 15710 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:10.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.984028 15710 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:10.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:10.987551 15708 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } attempt: 4
03:08:10.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.992792 15708 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.61:40499: Network error: Client connection negotiation failed: client connection to 127.15.67.61:40499: connect: Connection refused (error 111)
03:08:10.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.997488 15719 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:11.292377 15708 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } attempt: 5
03:08:11.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:11.299366 15708 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.61:40499: Network error: Client connection negotiation failed: client connection to 127.15.67.61:40499: connect: Connection refused (error 111)
03:08:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:11.847095 15708 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } attempt: 6
03:08:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:11.860833 15708 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.61:40499: Network error: Client connection negotiation failed: client connection to 127.15.67.61:40499: connect: Connection refused (error 111)
03:08:12.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:11.003638 15722 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:12.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:10.998731 15720 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:12.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:12.387514 15721 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1370 milliseconds
03:08:12.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.387673 15710 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:12.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.391913 15710 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:12.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.395210 15710 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.396920 15710 hybrid_clock.cc:648] HybridClock initialized: now 1774062492396682 us; error 102 us; skew 500 ppm
03:08:12.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.398038 15710 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:12.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.409173 15710 webserver.cc:492] Webserver started at http://127.15.67.61:43761/ using document root <none> and password file <none>
03:08:12.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.411510 15710 fs_manager.cc:362] Metadata directory not provided
03:08:12.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.411849 15710 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:12.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.412353 15710 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:08:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.417667 15710 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-1/data/instance:
03:08:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7585643fdba741f1ba8035fb821342a2"
03:08:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:12 on dist-test-slave-62xr"
03:08:12.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.418920 15710 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-1/wal/instance:
03:08:12.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7585643fdba741f1ba8035fb821342a2"
03:08:12.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:12 on dist-test-slave-62xr"
03:08:12.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.428215 15710 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.006s sys 0.001s
03:08:12.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.436409 15731 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.438809 15710 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.001s sys 0.004s
03:08:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.439374 15710 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-1/data,/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-1/wal
03:08:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7585643fdba741f1ba8035fb821342a2"
03:08:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:12 on dist-test-slave-62xr"
03:08:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.439831 15710 fs_report.cc:389] FS layout report
03:08:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-1/wal
03:08:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-1/wal
03:08:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-1/data/data
03:08:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:12.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.496100 15710 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:12.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.497748 15710 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:12.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.498309 15710 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:12.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.588594 15710 rpc_server.cc:307] RPC server started. Bound to: 127.15.67.61:40499
03:08:12.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.588676 15782 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.67.61:40499 every 8 connection(s)
03:08:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.591823 15710 server_base.cc:1193] Dumped server information to /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-1/data/info.pb
03:08:12.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.598706 15783 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:08:12.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.598919 15628 external_mini_cluster.cc:1428] Started /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu as pid 15710
03:08:12.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.599411 15628 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-1/wal/instance
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.602720 15628 external_mini_cluster.cc:1366] Running /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-2/wal
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-2/data
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-2/logs
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-2/data/info.pb
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.67.60:46163
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.67.60
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.67.20:40279
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.67.62:38603,127.15.67.61:40499,127.15.67.60:46163
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44009/jwks.json with env {}
03:08:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.614616 15783 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } has no permanent_uuid. Determining permanent_uuid...
03:08:12.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.638955 15783 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } has no permanent_uuid. Determining permanent_uuid...
03:08:12.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.656672 15783 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } has no permanent_uuid. Determining permanent_uuid...
03:08:12.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:12.659982 15732 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.67.60:46163: connect: Connection refused (error 111)
03:08:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:12.664669 15783 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.60:46163: Network error: Client connection negotiation failed: client connection to 127.15.67.60:46163: connect: Connection refused (error 111)
03:08:12.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.714408 15783 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } attempt: 1
03:08:12.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:12.718953 15783 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.60:46163: Network error: Client connection negotiation failed: client connection to 127.15.67.60:46163: connect: Connection refused (error 111)
03:08:12.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.787520 15783 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } attempt: 2
03:08:12.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:12.791904 15783 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.60:46163: Network error: Client connection negotiation failed: client connection to 127.15.67.60:46163: connect: Connection refused (error 111)
03:08:12.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.883502 15783 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } attempt: 3
03:08:12.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:12.888242 15783 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.60:46163: Network error: Client connection negotiation failed: client connection to 127.15.67.60:46163: connect: Connection refused (error 111)
03:08:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.921514 15708 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } attempt: 7
03:08:12.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:12.936779 15708 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } has no permanent_uuid. Determining permanent_uuid...
03:08:12.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:12.944289 15708 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.60:46163: Network error: Client connection negotiation failed: client connection to 127.15.67.60:46163: connect: Connection refused (error 111)
03:08:12.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:12.963896 15785 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:12.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:12.964483 15785 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:12.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:12.964982 15785 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:13.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:13.001130 15785 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:13.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:13.001452 15785 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:13.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:13.001672 15785 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:13.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:13.001899 15785 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:13.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:13.002121 15785 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:13.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:13.002272 15785 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44009/jwks.json
03:08:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:13.003104 15708 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } attempt: 1
03:08:13.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:13.009709 15708 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.60:46163: Network error: Client connection negotiation failed: client connection to 127.15.67.60:46163: connect: Connection refused (error 111)
03:08:13.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:13.031847 15783 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } attempt: 4
03:08:13.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:13.038679 15783 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.60:46163: Network error: Client connection negotiation failed: client connection to 127.15.67.60:46163: connect: Connection refused (error 111)
03:08:13.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:13.050035 15785 master_runner.cc:386] Master server non-default flags:
03:08:13.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:13.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.67.20:40279
03:08:13.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:13.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:13.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-2/data
03:08:13.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-2/wal
03:08:13.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:13.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.67.62:38603,127.15.67.61:40499,127.15.67.60:46163
03:08:13.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:13.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:13.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.67.60:46163
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44009/jwks.json
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-2/data/info.pb
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.67.60
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-2/logs
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 68f10dd41067059260a5fb5c725b178b69c38aed
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Mar 2026 02:43:22 UTC on e7f111948823
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11031
03:08:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:13.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:13.051666 15785 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:13.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:13.053787 15785 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:13.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:13.066740 15797 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:13.091256 15708 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } attempt: 2
03:08:13.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:13.096735 15708 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.60:46163: Network error: Client connection negotiation failed: client connection to 127.15.67.60:46163: connect: Connection refused (error 111)
03:08:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:13.182299 15708 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } attempt: 3
03:08:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:13.196332 15708 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.60:46163: Network error: Client connection negotiation failed: client connection to 127.15.67.60:46163: connect: Connection refused (error 111)
03:08:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:13.337393 15708 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } attempt: 4
03:08:13.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:13.338303 15783 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } attempt: 5
03:08:13.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:13.347046 15783 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.60:46163: Network error: Client connection negotiation failed: client connection to 127.15.67.60:46163: connect: Connection refused (error 111)
03:08:13.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:13.349567 15708 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.60:46163: Network error: Client connection negotiation failed: client connection to 127.15.67.60:46163: connect: Connection refused (error 111)
03:08:13.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:13.633853 15708 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } attempt: 5
03:08:13.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:13.643970 15708 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.60:46163: Network error: Client connection negotiation failed: client connection to 127.15.67.60:46163: connect: Connection refused (error 111)
03:08:13.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:13.895419 15783 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } attempt: 6
03:08:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:13.929914 15783 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.60:46163: Network error: Client connection negotiation failed: client connection to 127.15.67.60:46163: connect: Connection refused (error 111)
03:08:14.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.196704 15708 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } attempt: 6
03:08:14.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:14.215082 15708 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.67.60:46163: Network error: Client connection negotiation failed: client connection to 127.15.67.60:46163: connect: Connection refused (error 111)
03:08:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:13.067224 15798 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:14.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:14.671389 15800 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:14.676797 15799 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1602 milliseconds
03:08:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.676812 15785 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:14.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.678458 15785 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:14.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.681444 15785 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:14.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.683085 15785 hybrid_clock.cc:648] HybridClock initialized: now 1774062494682892 us; error 180 us; skew 500 ppm
03:08:14.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.685307 15785 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:14.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.694747 15785 webserver.cc:492] Webserver started at http://127.15.67.60:39697/ using document root <none> and password file <none>
03:08:14.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.696379 15785 fs_manager.cc:362] Metadata directory not provided
03:08:14.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.696722 15785 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:14.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.697381 15785 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:08:14.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.702421 15785 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-2/data/instance:
03:08:14.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "105ef371182b4c7a8a5dad0ba739a00d"
03:08:14.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:14 on dist-test-slave-62xr"
03:08:14.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.704807 15785 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-2/wal/instance:
03:08:14.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "105ef371182b4c7a8a5dad0ba739a00d"
03:08:14.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:14 on dist-test-slave-62xr"
03:08:14.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.716105 15785 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.006s sys 0.005s
03:08:14.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.723807 15811 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:14.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.725215 15785 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
03:08:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.725844 15785 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-2/data,/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-2/wal
03:08:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "105ef371182b4c7a8a5dad0ba739a00d"
03:08:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:14 on dist-test-slave-62xr"
03:08:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.726274 15785 fs_report.cc:389] FS layout report
03:08:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-2/wal
03:08:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-2/wal
03:08:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-2/data/data
03:08:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:14.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.809897 15785 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:14.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.811939 15785 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:14.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.812557 15785 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:14.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.897862 15785 rpc_server.cc:307] RPC server started. Bound to: 127.15.67.60:46163
03:08:14.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.897948 15862 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.67.60:46163 every 8 connection(s)
03:08:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.901440 15785 server_base.cc:1193] Dumped server information to /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-2/data/info.pb
03:08:14.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.902689 15628 external_mini_cluster.cc:1428] Started /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu as pid 15785
03:08:14.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.903252 15628 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/master-2/wal/instance
03:08:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.911726 15863 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:08:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.932771 15863 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } has no permanent_uuid. Determining permanent_uuid...
03:08:14.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.960992 15863 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } has no permanent_uuid. Determining permanent_uuid...
03:08:14.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.972448 15863 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } has no permanent_uuid. Determining permanent_uuid...
03:08:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.991991 15783 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } attempt: 7
03:08:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:14.995215 15863 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d: Bootstrap starting.
03:08:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.006816 15863 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d: Neither blocks nor log segments found. Creating new log.
03:08:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.011664 15863 log.cc:826] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d: Log is configured to *not* fsync() on all Append() calls
03:08:15.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.020538 15783 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2: Bootstrap starting.
03:08:15.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.021926 15863 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d: No bootstrap required, opened a new log
03:08:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.027544 15783 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2: Neither blocks nor log segments found. Creating new log.
03:08:15.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.031397 15783 log.cc:826] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2: Log is configured to *not* fsync() on all Append() calls
03:08:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.037111 15783 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2: No bootstrap required, opened a new log
03:08:15.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.049326 15863 raft_consensus.cc:359] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } }
03:08:15.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.049991 15863 raft_consensus.cc:385] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:08:15.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.050204 15863 raft_consensus.cc:740] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 105ef371182b4c7a8a5dad0ba739a00d, State: Initialized, Role: FOLLOWER
03:08:15.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.050884 15863 consensus_queue.cc:260] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } }
03:08:15.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.054032 15873 sys_catalog.cc:455] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } } }
03:08:15.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.054945 15873 sys_catalog.cc:458] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d [sys.catalog]: This master's current role is: FOLLOWER
03:08:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.056053 15863 sys_catalog.cc:565] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d [sys.catalog]: configured and running, proceeding with master startup.
03:08:15.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.058419 15783 raft_consensus.cc:359] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } }
03:08:15.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.059540 15783 raft_consensus.cc:385] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:08:15.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.059990 15783 raft_consensus.cc:740] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7585643fdba741f1ba8035fb821342a2, State: Initialized, Role: FOLLOWER
03:08:15.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.061293 15783 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } }
03:08:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.064797 15877 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } } }
03:08:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.065898 15877 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [sys.catalog]: This master's current role is: FOLLOWER
03:08:15.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.066848 15783 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [sys.catalog]: configured and running, proceeding with master startup.
03:08:15.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:15.068382 15817 tablet.cc:2379] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:08:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:15.110090 15890 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:08:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:15.110807 15890 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:08:15.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.114022 15877 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:08:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.114764 15877 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } }
03:08:15.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.120496 15877 leader_election.cc:290] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 453ece4e574d4392bc70917f43a42108 (127.15.67.62:38603), 105ef371182b4c7a8a5dad0ba739a00d (127.15.67.60:46163)
03:08:15.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.121483 15838 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7585643fdba741f1ba8035fb821342a2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "105ef371182b4c7a8a5dad0ba739a00d" is_pre_election: true
03:08:15.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.122561 15838 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7585643fdba741f1ba8035fb821342a2 in term 0.
03:08:15.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.124200 15732 leader_election.cc:304] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 105ef371182b4c7a8a5dad0ba739a00d, 7585643fdba741f1ba8035fb821342a2; no voters:
03:08:15.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.125272 15877 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:08:15.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.125654 15877 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:08:15.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.126039 15877 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [term 0 FOLLOWER]: Advancing to term 1
03:08:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.134130 15877 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } }
03:08:15.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.139066 15877 leader_election.cc:290] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [CANDIDATE]: Term 1 election: Requested vote from peers 453ece4e574d4392bc70917f43a42108 (127.15.67.62:38603), 105ef371182b4c7a8a5dad0ba739a00d (127.15.67.60:46163)
03:08:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.140072 15838 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7585643fdba741f1ba8035fb821342a2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "105ef371182b4c7a8a5dad0ba739a00d"
03:08:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.140658 15838 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d [term 0 FOLLOWER]: Advancing to term 1
03:08:15.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.094326 15785 master_runner.cc:186] Error getting master registration for 127.15.67.62:38603: OK, instance_id { permanent_uuid: "453ece4e574d4392bc70917f43a42108" instance_seqno: 1774062490534062 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:08:15.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.147095 15785 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:15.152895 15897 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:08:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:15.153347 15897 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:08:15.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.170499 15838 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7585643fdba741f1ba8035fb821342a2 in term 1.
03:08:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.171989 15732 leader_election.cc:304] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 105ef371182b4c7a8a5dad0ba739a00d, 7585643fdba741f1ba8035fb821342a2; no voters:
03:08:15.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.173223 15877 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [term 1 FOLLOWER]: Leader election won for term 1
03:08:15.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.142501 15710 master_runner.cc:186] Error getting master registration for 127.15.67.62:38603: OK, instance_id { permanent_uuid: "453ece4e574d4392bc70917f43a42108" instance_seqno: 1774062490534062 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:08:15.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.177858 15710 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:15.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.180477 15877 raft_consensus.cc:697] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [term 1 LEADER]: Becoming Leader. State: Replica: 7585643fdba741f1ba8035fb821342a2, State: Running, Role: LEADER
03:08:15.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.181830 15877 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } }
03:08:15.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.201500 15899 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7585643fdba741f1ba8035fb821342a2. Latest consensus state: current_term: 1 leader_uuid: "7585643fdba741f1ba8035fb821342a2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } } }
03:08:15.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.202397 15899 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [sys.catalog]: This master's current role is: LEADER
03:08:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.207700 15901 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:08:15.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.216063 15901 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:08:15.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.240079 15838 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d [term 1 FOLLOWER]: Refusing update from remote peer 7585643fdba741f1ba8035fb821342a2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:08:15.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.242308 15899 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
03:08:15.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.248770 15708 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } attempt: 7
03:08:15.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:15.259418 15737 tablet.cc:2379] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:08:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.289373 15708 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108: Bootstrap starting.
03:08:15.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.318643 15708 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108: Neither blocks nor log segments found. Creating new log.
03:08:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.321564 15708 log.cc:826] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108: Log is configured to *not* fsync() on all Append() calls
03:08:15.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.324797 15902 mvcc.cc:204] Tried to move back new op lower bound from 7266559980482101248 to 7266559980277342208. Current Snapshot: MvccSnapshot[applied={T|T < 7266559980482101248}]
03:08:15.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.327942 15873 sys_catalog.cc:455] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7585643fdba741f1ba8035fb821342a2. Latest consensus state: current_term: 1 leader_uuid: "7585643fdba741f1ba8035fb821342a2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } } }
03:08:15.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.328889 15873 sys_catalog.cc:458] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d [sys.catalog]: This master's current role is: FOLLOWER
03:08:15.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.335831 15708 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108: No bootstrap required, opened a new log
03:08:15.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.335711 15904 mvcc.cc:204] Tried to move back new op lower bound from 7266559980482101248 to 7266559980277342208. Current Snapshot: MvccSnapshot[applied={T|T < 7266559980482101248}]
03:08:15.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.348376 15877 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7585643fdba741f1ba8035fb821342a2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } } }
03:08:15.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.349246 15877 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [sys.catalog]: This master's current role is: LEADER
03:08:15.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.350564 15873 sys_catalog.cc:455] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7585643fdba741f1ba8035fb821342a2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } } }
03:08:15.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.351295 15873 sys_catalog.cc:458] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d [sys.catalog]: This master's current role is: FOLLOWER
03:08:15.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.367224 15901 catalog_manager.cc:1357] Generated new cluster ID: 3f3eb8ce895446d7b8620cb31cba13b7
03:08:15.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.367596 15901 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:08:15.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.378743 15708 raft_consensus.cc:359] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } }
03:08:15.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.379729 15708 raft_consensus.cc:385] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:08:15.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.380059 15708 raft_consensus.cc:740] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 453ece4e574d4392bc70917f43a42108, State: Initialized, Role: FOLLOWER
03:08:15.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.381217 15708 consensus_queue.cc:260] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } }
03:08:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.396023 15909 sys_catalog.cc:455] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } } }
03:08:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.397220 15909 sys_catalog.cc:458] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108 [sys.catalog]: This master's current role is: FOLLOWER
03:08:15.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.398581 15708 sys_catalog.cc:565] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108 [sys.catalog]: configured and running, proceeding with master startup.
03:08:15.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.400836 15682 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7585643fdba741f1ba8035fb821342a2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "453ece4e574d4392bc70917f43a42108"
03:08:15.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.402319 15682 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108 [term 0 FOLLOWER]: Advancing to term 1
03:08:15.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.408057 15683 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7585643fdba741f1ba8035fb821342a2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "453ece4e574d4392bc70917f43a42108" is_pre_election: true
03:08:15.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.422585 15682 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7585643fdba741f1ba8035fb821342a2 in term 1.
03:08:15.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.431203 15681 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108 [term 1 FOLLOWER]: Refusing update from remote peer 7585643fdba741f1ba8035fb821342a2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:08:15.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.446002 15899 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
03:08:15.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.506901 15901 catalog_manager.cc:1380] Generated new certificate authority record
03:08:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.510103 15901 catalog_manager.cc:1514] Loading token signing keys...
03:08:15.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.520442 15909 sys_catalog.cc:455] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7585643fdba741f1ba8035fb821342a2. Latest consensus state: current_term: 1 leader_uuid: "7585643fdba741f1ba8035fb821342a2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } } }
03:08:15.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.520460 15877 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7585643fdba741f1ba8035fb821342a2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } } }
03:08:15.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.521204 15909 sys_catalog.cc:458] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108 [sys.catalog]: This master's current role is: FOLLOWER
03:08:15.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.521243 15877 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2 [sys.catalog]: This master's current role is: LEADER
03:08:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.521880 15921 sys_catalog.cc:455] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7585643fdba741f1ba8035fb821342a2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "453ece4e574d4392bc70917f43a42108" member_type: VOTER last_known_addr { host: "127.15.67.62" port: 38603 } } peers { permanent_uuid: "7585643fdba741f1ba8035fb821342a2" member_type: VOTER last_known_addr { host: "127.15.67.61" port: 40499 } } peers { permanent_uuid: "105ef371182b4c7a8a5dad0ba739a00d" member_type: VOTER last_known_addr { host: "127.15.67.60" port: 46163 } } }
03:08:15.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.527424 15921 sys_catalog.cc:458] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108 [sys.catalog]: This master's current role is: FOLLOWER
03:08:15.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.583953 15926 catalog_manager.cc:1269] Loaded cluster ID: 3f3eb8ce895446d7b8620cb31cba13b7
03:08:15.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.584249 15926 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108: loading cluster ID for follower catalog manager: success
03:08:15.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.589495 15901 catalog_manager.cc:6044] T 00000000000000000000000000000000 P 7585643fdba741f1ba8035fb821342a2: Generated new TSK 0
03:08:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.590864 15901 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:08:15.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.594970 15926 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108: acquiring CA information for follower catalog manager: success
03:08:15.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.605403 15926 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 453ece4e574d4392bc70917f43a42108: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:15.619980 15628 external_mini_cluster.cc:1366] Running /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-0/wal
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-0/data
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-0/logs
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-0/data/info.pb
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:15.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.67.1:0
03:08:15.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.67.1
03:08:15.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.67.1
03:08:15.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:08:15.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.67.62:38603,127.15.67.61:40499,127.15.67.60:46163
03:08:15.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.67.20:40279
03:08:15.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:15.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:15.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:08:16.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:16.117089 15890 catalog_manager.cc:1269] Loaded cluster ID: 3f3eb8ce895446d7b8620cb31cba13b7
03:08:16.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:16.117844 15890 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d: loading cluster ID for follower catalog manager: success
03:08:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:16.133976 15890 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d: acquiring CA information for follower catalog manager: success
03:08:16.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:16.154402 15890 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 105ef371182b4c7a8a5dad0ba739a00d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:16.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:16.265141 15928 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:16.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:16.265843 15928 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:16.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:16.266664 15928 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:16.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:16.336983 15928 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:16.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:16.338893 15928 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.15.67.1
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:16.415488 15928 tablet_server_runner.cc:78] Tablet server non-default flags:
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.67.20:40279
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-0/data
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-0/wal
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.67.1:0
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-0/data/info.pb
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.67.1
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.67.62:38603,127.15.67.61:40499,127.15.67.60:46163
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.67.1
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-0/logs
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 68f10dd41067059260a5fb5c725b178b69c38aed
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Mar 2026 02:43:22 UTC on e7f111948823
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11031
03:08:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:16.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:16.417508 15928 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:16.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:16.420122 15928 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:16.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:16.459362 15934 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:19.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:17.851713 15933 debug-util.cc:398] Leaking SignalData structure 0x7b080002eda0 after lost signal to thread 15928
03:08:19.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:19.251549 15933 kernel_stack_watchdog.cc:198] Thread 15928 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 378ms:
03:08:19.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:08:19.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:08:19.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:08:19.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:08:19.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:16.475708 15935 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:19.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:19.255826 15928 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.785s user 0.943s sys 1.390s
03:08:19.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:19.256731 15928 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.786s user 0.943s sys 1.390s
03:08:19.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.268383 15928 server_base.cc:1061] running on GCE node
03:08:19.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:19.270728 15941 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:19.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.272529 15928 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:19.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.286465 15928 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:19.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.288306 15928 hybrid_clock.cc:648] HybridClock initialized: now 1774062499288264 us; error 50 us; skew 500 ppm
03:08:19.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.289656 15928 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:19.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.332255 15928 webserver.cc:492] Webserver started at http://127.15.67.1:40649/ using document root <none> and password file <none>
03:08:19.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.334000 15928 fs_manager.cc:362] Metadata directory not provided
03:08:19.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.334488 15928 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:19.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.335203 15928 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:08:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.342262 15928 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-0/data/instance:
03:08:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "526c5a82c3d446b98940e66fa6b431a6"
03:08:19.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:19 on dist-test-slave-62xr"
03:08:19.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.344218 15928 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-0/wal/instance:
03:08:19.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "526c5a82c3d446b98940e66fa6b431a6"
03:08:19.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:19 on dist-test-slave-62xr"
03:08:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.370625 15928 fs_manager.cc:696] Time spent creating directory manager: real 0.025s user 0.012s sys 0.012s
03:08:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.385426 15946 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:19.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.387502 15928 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.005s sys 0.004s
03:08:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.388255 15928 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-0/data,/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-0/wal
03:08:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "526c5a82c3d446b98940e66fa6b431a6"
03:08:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:19 on dist-test-slave-62xr"
03:08:19.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.389142 15928 fs_report.cc:389] FS layout report
03:08:19.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:19.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-0/wal
03:08:19.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-0/wal
03:08:19.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-0/data/data
03:08:19.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:19.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:19.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:19.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:19.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:19.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:19.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:19.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:19.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:19.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:19.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.498000 15928 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:19.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.500311 15928 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:19.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.501194 15928 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:19.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.510744 15928 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:08:19.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.520820 15928 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:08:19.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.521136 15928 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:08:19.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.521471 15928 ts_tablet_manager.cc:616] Registered 0 tablets
03:08:19.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.521678 15928 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:08:19.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.767745 15928 rpc_server.cc:307] RPC server started. Bound to: 127.15.67.1:44045
03:08:19.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.767863 16058 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.67.1:44045 every 8 connection(s)
03:08:19.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.770865 15928 server_base.cc:1193] Dumped server information to /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-0/data/info.pb
03:08:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.780694 15628 external_mini_cluster.cc:1428] Started /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu as pid 15928
03:08:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.781572 15628 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-0/wal/instance
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.794299 15628 external_mini_cluster.cc:1366] Running /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-1/wal
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-1/data
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-1/logs
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-1/data/info.pb
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.67.2:0
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.67.2
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.67.2
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.67.62:38603,127.15.67.61:40499,127.15.67.60:46163
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.67.20:40279
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:08:19.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.835175 16060 heartbeater.cc:344] Connected to a master server at 127.15.67.61:40499
03:08:19.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.835844 16060 heartbeater.cc:461] Registering TS with master...
03:08:19.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.836647 16059 heartbeater.cc:344] Connected to a master server at 127.15.67.60:46163
03:08:19.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.837073 16059 heartbeater.cc:461] Registering TS with master...
03:08:19.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.837365 16060 heartbeater.cc:507] Master 127.15.67.61:40499 requested a full tablet report, sending...
03:08:19.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.838424 16059 heartbeater.cc:507] Master 127.15.67.60:46163 requested a full tablet report, sending...
03:08:19.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.841415 15748 ts_manager.cc:194] Registered new tserver with Master: 526c5a82c3d446b98940e66fa6b431a6 (127.15.67.1:44045)
03:08:19.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.841660 15828 ts_manager.cc:194] Registered new tserver with Master: 526c5a82c3d446b98940e66fa6b431a6 (127.15.67.1:44045)
03:08:19.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.843838 16061 heartbeater.cc:344] Connected to a master server at 127.15.67.62:38603
03:08:19.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.844252 16061 heartbeater.cc:461] Registering TS with master...
03:08:19.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.844561 15748 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.15.67.1:36207
03:08:19.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.845175 16061 heartbeater.cc:507] Master 127.15.67.62:38603 requested a full tablet report, sending...
03:08:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:19.849604 15673 ts_manager.cc:194] Registered new tserver with Master: 526c5a82c3d446b98940e66fa6b431a6 (127.15.67.1:44045)
03:08:20.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:20.187273 16069 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:20.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:20.187909 16069 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:20.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:20.188690 16069 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:20.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:20.229344 16069 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:20.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:20.230716 16069 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.15.67.2
03:08:20.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:20.271715 16069 tablet_server_runner.cc:78] Tablet server non-default flags:
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.67.20:40279
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-1/data
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-1/wal
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.67.2:0
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-1/data/info.pb
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.67.2
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.67.62:38603,127.15.67.61:40499,127.15.67.60:46163
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.67.2
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-1/logs
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 68f10dd41067059260a5fb5c725b178b69c38aed
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Mar 2026 02:43:22 UTC on e7f111948823
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11031
03:08:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:20.273476 16069 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:20.275583 16069 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:20.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:20.290885 16075 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:20.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:20.849192 16060 heartbeater.cc:499] Master 127.15.67.61:40499 was elected leader, sending a full tablet report...
03:08:21.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:20.297986 16078 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:21.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:20.292613 16076 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:21.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:21.677261 16077 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1381 milliseconds
03:08:21.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.677448 16069 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:21.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.679000 16069 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:21.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.681438 16069 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:21.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.682947 16069 hybrid_clock.cc:648] HybridClock initialized: now 1774062501682896 us; error 39 us; skew 500 ppm
03:08:21.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.684013 16069 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:21.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.691326 16069 webserver.cc:492] Webserver started at http://127.15.67.2:46089/ using document root <none> and password file <none>
03:08:21.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.692502 16069 fs_manager.cc:362] Metadata directory not provided
03:08:21.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.692767 16069 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:21.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.693284 16069 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:08:21.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.697928 16069 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-1/data/instance:
03:08:21.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3868f67143af4bc8973a9eb63b243c1c"
03:08:21.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:21 on dist-test-slave-62xr"
03:08:21.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.699229 16069 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-1/wal/instance:
03:08:21.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3868f67143af4bc8973a9eb63b243c1c"
03:08:21.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:21 on dist-test-slave-62xr"
03:08:21.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.706827 16069 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
03:08:21.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.713059 16086 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.714524 16069 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
03:08:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.714936 16069 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-1/data,/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-1/wal
03:08:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3868f67143af4bc8973a9eb63b243c1c"
03:08:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:21 on dist-test-slave-62xr"
03:08:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.715335 16069 fs_report.cc:389] FS layout report
03:08:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-1/wal
03:08:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-1/wal
03:08:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-1/data/data
03:08:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:21.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.769445 16069 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:21.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.771603 16069 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:21.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.772228 16069 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:21.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.775861 16069 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:08:21.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.782406 16069 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:08:21.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.782718 16069 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:08:21.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.783022 16069 ts_tablet_manager.cc:616] Registered 0 tablets
03:08:21.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.783245 16069 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:08:21.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.951788 16069 rpc_server.cc:307] RPC server started. Bound to: 127.15.67.2:38677
03:08:21.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.951881 16198 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.67.2:38677 every 8 connection(s)
03:08:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.955967 16069 server_base.cc:1193] Dumped server information to /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-1/data/info.pb
03:08:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.966411 15628 external_mini_cluster.cc:1428] Started /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu as pid 16069
03:08:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.966940 15628 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-1/wal/instance
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.992419 15628 external_mini_cluster.cc:1366] Running /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-2/wal
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-2/data
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-2/logs
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-2/data/info.pb
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:08:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:21.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.67.3:0
03:08:21.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.67.3
03:08:21.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.67.3
03:08:21.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:08:21.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.67.62:38603,127.15.67.61:40499,127.15.67.60:46163
03:08:21.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.67.20:40279
03:08:21.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:21.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:21.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:08:21.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.998592 16201 heartbeater.cc:344] Connected to a master server at 127.15.67.62:38603
03:08:21.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:21.999212 16201 heartbeater.cc:461] Registering TS with master...
03:08:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:22.000919 16201 heartbeater.cc:507] Master 127.15.67.62:38603 requested a full tablet report, sending...
03:08:22.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:22.003700 16200 heartbeater.cc:344] Connected to a master server at 127.15.67.61:40499
03:08:22.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:22.004421 16200 heartbeater.cc:461] Registering TS with master...
03:08:22.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:22.003880 16199 heartbeater.cc:344] Connected to a master server at 127.15.67.60:46163
03:08:22.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:22.005877 16199 heartbeater.cc:461] Registering TS with master...
03:08:22.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:22.006983 15673 ts_manager.cc:194] Registered new tserver with Master: 3868f67143af4bc8973a9eb63b243c1c (127.15.67.2:38677)
03:08:22.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:22.007448 16200 heartbeater.cc:507] Master 127.15.67.61:40499 requested a full tablet report, sending...
03:08:22.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:22.007505 16199 heartbeater.cc:507] Master 127.15.67.60:46163 requested a full tablet report, sending...
03:08:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:22.011027 15748 ts_manager.cc:194] Registered new tserver with Master: 3868f67143af4bc8973a9eb63b243c1c (127.15.67.2:38677)
03:08:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:22.011314 15828 ts_manager.cc:194] Registered new tserver with Master: 3868f67143af4bc8973a9eb63b243c1c (127.15.67.2:38677)
03:08:22.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:22.014773 15748 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.15.67.2:41495
03:08:22.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:22.378870 15779 debug-util.cc:398] Leaking SignalData structure 0x7b0800096e80 after lost signal to thread 15714
03:08:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:22.392889 16209 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:22.393534 16209 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:22.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:22.394006 16209 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:22.442747 16209 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:22.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:22.443840 16209 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.15.67.3
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:22.488823 16209 tablet_server_runner.cc:78] Tablet server non-default flags:
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.67.20:40279
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-2/data
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-2/wal
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.67.3:0
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-2/data/info.pb
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.67.3
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.67.62:38603,127.15.67.61:40499,127.15.67.60:46163
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.67.3
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-2/logs
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:22.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:08:22.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:22.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 68f10dd41067059260a5fb5c725b178b69c38aed
03:08:22.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:22.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Mar 2026 02:43:22 UTC on e7f111948823
03:08:22.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11031
03:08:22.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:22.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:22.490650 16209 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:22.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:22.492581 16209 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:22.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:22.509727 16215 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:23.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:23.020134 16200 heartbeater.cc:499] Master 127.15.67.61:40499 was elected leader, sending a full tablet report...
03:08:24.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:22.514700 16218 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:24.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:22.510972 16216 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:24.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:24.169860 16217 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1651 milliseconds
03:08:24.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.169992 16209 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:24.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.171716 16209 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:24.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.174909 16209 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:24.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.176697 16209 hybrid_clock.cc:648] HybridClock initialized: now 1774062504176649 us; error 62 us; skew 500 ppm
03:08:24.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.177935 16209 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:24.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.186672 16209 webserver.cc:492] Webserver started at http://127.15.67.3:34407/ using document root <none> and password file <none>
03:08:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.188110 16209 fs_manager.cc:362] Metadata directory not provided
03:08:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.188421 16209 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.189011 16209 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:08:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.196349 16209 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-2/data/instance:
03:08:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4ffcf642a8044e168cbcb9363b3c00b6"
03:08:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:24 on dist-test-slave-62xr"
03:08:24.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.198247 16209 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-2/wal/instance:
03:08:24.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4ffcf642a8044e168cbcb9363b3c00b6"
03:08:24.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:24 on dist-test-slave-62xr"
03:08:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.209888 16209 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.013s sys 0.000s
03:08:24.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.219591 16226 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:24.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.221410 16209 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.003s sys 0.001s
03:08:24.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.221908 16209 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-2/data,/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-2/wal
03:08:24.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4ffcf642a8044e168cbcb9363b3c00b6"
03:08:24.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:24 on dist-test-slave-62xr"
03:08:24.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.222528 16209 fs_report.cc:389] FS layout report
03:08:24.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:24.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-2/wal
03:08:24.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-2/wal
03:08:24.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-2/data/data
03:08:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:24.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.287596 16209 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:24.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.289786 16209 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:24.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.290337 16209 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.294193 16209 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:08:24.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.300884 16209 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:08:24.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.301117 16209 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:08:24.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.301354 16209 ts_tablet_manager.cc:616] Registered 0 tablets
03:08:24.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.301499 16209 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:08:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.497624 16209 rpc_server.cc:307] RPC server started. Bound to: 127.15.67.3:32795
03:08:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.497931 16338 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.67.3:32795 every 8 connection(s)
03:08:24.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.502113 16209 server_base.cc:1193] Dumped server information to /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-2/data/info.pb
03:08:24.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.511170 15628 external_mini_cluster.cc:1428] Started /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu as pid 16209
03:08:24.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.512805 15628 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster9280425016825571461/ts-2/wal/instance
03:08:24.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.554736 16339 heartbeater.cc:344] Connected to a master server at 127.15.67.60:46163
03:08:24.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.555800 16339 heartbeater.cc:461] Registering TS with master...
03:08:24.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.557255 16339 heartbeater.cc:507] Master 127.15.67.60:46163 requested a full tablet report, sending...
03:08:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.558573 16340 heartbeater.cc:344] Connected to a master server at 127.15.67.61:40499
03:08:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.558951 16340 heartbeater.cc:461] Registering TS with master...
03:08:24.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.559931 16340 heartbeater.cc:507] Master 127.15.67.61:40499 requested a full tablet report, sending...
03:08:24.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.561610 16341 heartbeater.cc:344] Connected to a master server at 127.15.67.62:38603
03:08:24.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.561939 16341 heartbeater.cc:461] Registering TS with master...
03:08:24.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.562906 16341 heartbeater.cc:507] Master 127.15.67.62:38603 requested a full tablet report, sending...
03:08:24.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.563558 15828 ts_manager.cc:194] Registered new tserver with Master: 4ffcf642a8044e168cbcb9363b3c00b6 (127.15.67.3:32795)
03:08:24.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.565392 15748 ts_manager.cc:194] Registered new tserver with Master: 4ffcf642a8044e168cbcb9363b3c00b6 (127.15.67.3:32795)
03:08:24.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.566356 15673 ts_manager.cc:194] Registered new tserver with Master: 4ffcf642a8044e168cbcb9363b3c00b6 (127.15.67.3:32795)
03:08:24.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.567238 15748 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.15.67.3:50611
03:08:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:24.579924 15628 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
03:08:24.594 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:08:24.594 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
03:08:24.596 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.15.67.62"
port: 38603
}
bound_http_address {
host: "127.15.67.62"
port: 33889
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.15.67.61"
port: 40499
}
bound_http_address {
host: "127.15.67.61"
port: 43761
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.15.67.60"
port: 46163
}
bound_http_address {
host: "127.15.67.60"
port: 39697
}
}
}
03:08:24.596 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:08:24.597 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.15.67.1"
port: 44045
}
bound_http_address {
host: "127.15.67.1"
port: 40649
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.15.67.2"
port: 38677
}
bound_http_address {
host: "127.15.67.2"
port: 46089
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.15.67.3"
port: 32795
}
bound_http_address {
host: "127.15.67.3"
port: 34407
}
}
}
03:08:24.597 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:08:24.599 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster16677344248858439489/krb5kdc/krb5cc
03:08:24.601 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:08:24.602 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task67Ldfr/test-tmp
03:08:24.602 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:08:24.602 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task67Ldfr/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:08:24.612 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
03:08:25.067 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:25.068 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:08:25.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:25.074725 16350 mini_oidc.cc:150] Starting JWKS server
03:08:25.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:25.079576 16350 webserver.cc:492] Webserver started at http://127.0.0.1:44053/ using document root <none> and password file <none>
03:08:25.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:25.080230 16350 mini_oidc.cc:186] Starting OIDC Discovery server
03:08:25.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:25.082248 16350 webserver.cc:492] Webserver started at http://127.0.0.1:45523/ using document root <none> and password file <none>
03:08:25.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
03:08:25.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/principal' for realm 'KRBTEST.COM',
03:08:25.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
03:08:25.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:25 dist-test-slave-62xr krb5kdc[16362](info): setting up network...
03:08:25.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
03:08:25.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:25 dist-test-slave-62xr krb5kdc[16362](info): set up 2 sockets
03:08:25.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:25 dist-test-slave-62xr krb5kdc[16362](info): commencing operation
03:08:25.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
03:08:25.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:25.576200 16340 heartbeater.cc:499] Master 127.15.67.61:40499 was elected leader, sending a full tablet report...
03:08:27.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:27.183672 16195 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad960 after lost signal to thread 16070
03:08:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:27.185369 16195 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad860 after lost signal to thread 16198
03:08:27.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:27.519709 16350 mini_kdc.cc:121] Time spent starting KDC: real 2.389s user 0.010s sys 0.015s
03:08:27.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
03:08:27.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:08:27.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
03:08:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
03:08:27.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:08:27.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
03:08:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
03:08:27.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:08:27.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
03:08:27.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:08:27.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/test-user.keytab.
03:08:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/test-user.keytab.
03:08:27.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:27 dist-test-slave-62xr krb5kdc[16362](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062507, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:27.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:08:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-21T03:08:27Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
03:08:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-21T03:08:27Z Disabled control of system clock
03:08:27.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.247.190@KRBTEST.COM; defaulting to no policy
03:08:27.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:08:27.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.247.190@KRBTEST.COM" created.
03:08:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:08:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.247.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab.
03:08:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.247.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab.
03:08:27.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.15.247.190@KRBTEST.COM; defaulting to no policy
03:08:27.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:08:27.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.15.247.190@KRBTEST.COM" created.
03:08:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:08:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.15.247.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab.
03:08:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.15.247.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab.
03:08:27.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:27.822849 16350 external_mini_cluster.cc:1366] Running /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:27.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:27.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/wal
03:08:27.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data
03:08:27.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:27.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/logs
03:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data/info.pb
03:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.247.190:40563
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.247.190
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.247.190
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.247.148:37647
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.247.190:40563,127.15.247.189:35611,127.15.247.188:40735
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44053/jwks.json
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:28.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:28.260746 16379 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:28.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:28.261368 16379 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:28.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:28.262070 16379 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:28.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:28.300192 16379 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:28.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:28.300451 16379 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:28.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:28.300863 16379 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:28.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:28.301081 16379 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:28.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:28.301286 16379 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:28.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:28.301484 16379 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:28.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:28.301641 16379 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44053/jwks.json
03:08:28.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:28.354182 16379 master_runner.cc:386] Master server non-default flags:
03:08:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.247.148:37647
03:08:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data
03:08:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/wal
03:08:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.247.190:40563,127.15.247.189:35611,127.15.247.188:40735
03:08:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab
03:08:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.247.190
03:08:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.247.190:40563
03:08:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44053/jwks.json
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data/info.pb
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.247.190
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/logs
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 68f10dd41067059260a5fb5c725b178b69c38aed
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Mar 2026 02:43:22 UTC on e7f111948823
03:08:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11031
03:08:28.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:28.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:28.356527 16379 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:28.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:28.358031 16379 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:28.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:28.375291 16385 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:29.777513 16384 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 16379
03:08:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:30.609687 16384 kernel_stack_watchdog.cc:198] Thread 16379 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 396ms:
03:08:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:08:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:08:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:08:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:08:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:28.375792 16386 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:30.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:30.612159 16379 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.235s user 0.002s sys 0.000s
03:08:30.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:30.612638 16379 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.236s user 0.002s sys 0.000s
03:08:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:30.617102 16387 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2233 milliseconds
03:08:30.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:30.618754 16388 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:30.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.618875 16379 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:30.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.620133 16379 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:30.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.622673 16379 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.624064 16379 hybrid_clock.cc:648] HybridClock initialized: now 1774062510624027 us; error 75 us; skew 500 ppm
03:08:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.625182 16379 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:30.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:30 dist-test-slave-62xr krb5kdc[16362](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062510, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:30.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.631976 16379 init.cc:377] Logged in from keytab as kudu/127.15.247.190@KRBTEST.COM (short username kudu)
03:08:30.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.637404 16379 webserver.cc:492] Webserver started at http://127.15.247.190:45525/ using document root <none> and password file <none>
03:08:30.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.638521 16379 fs_manager.cc:362] Metadata directory not provided
03:08:30.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.638767 16379 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:30.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.639272 16379 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:08:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.645337 16379 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data/instance:
03:08:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d5c213d669ed47ed81f884b52df3c0b6"
03:08:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:30 on dist-test-slave-62xr"
03:08:30.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.647217 16379 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/wal/instance:
03:08:30.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d5c213d669ed47ed81f884b52df3c0b6"
03:08:30.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:30 on dist-test-slave-62xr"
03:08:30.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.656247 16379 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
03:08:30.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.662441 16399 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:30.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.663808 16379 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
03:08:30.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.664129 16379 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data,/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/wal
03:08:30.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d5c213d669ed47ed81f884b52df3c0b6"
03:08:30.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:30 on dist-test-slave-62xr"
03:08:30.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.664460 16379 fs_report.cc:389] FS layout report
03:08:30.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:30.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/wal
03:08:30.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/wal
03:08:30.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data/data
03:08:30.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:30.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:30.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:30.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:30.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:30.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:30.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:30.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:30.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:30.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:30.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:30.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.724663 16379 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:30.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.734860 16379 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:30.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.735498 16379 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:30.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.819198 16379 rpc_server.cc:307] RPC server started. Bound to: 127.15.247.190:40563
03:08:30.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.819303 16451 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.247.190:40563 every 8 connection(s)
03:08:30.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.822077 16379 server_base.cc:1193] Dumped server information to /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data/info.pb
03:08:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.827526 16350 external_mini_cluster.cc:1428] Started /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu as pid 16379
03:08:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.828032 16350 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/wal/instance
03:08:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.828667 16452 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:08:30.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.247.189@KRBTEST.COM; defaulting to no policy
03:08:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.845091 16452 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } has no permanent_uuid. Determining permanent_uuid...
03:08:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:08:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.247.189@KRBTEST.COM" created.
03:08:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:30 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062510, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.190@KRBTEST.COM for kudu/127.15.247.190@KRBTEST.COM
03:08:30.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:08:30.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.247.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab.
03:08:30.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.247.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab.
03:08:30.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.892822 16452 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } has no permanent_uuid. Determining permanent_uuid...
03:08:30.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:30.896132 16400 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111)
03:08:30.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.15.247.189@KRBTEST.COM; defaulting to no policy
03:08:30.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:08:30.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.15.247.189@KRBTEST.COM" created.
03:08:30.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:30.901659 16452 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.189:35611: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111)
03:08:30.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:08:30.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.15.247.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab.
03:08:30.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.15.247.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab.
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.929507 16350 external_mini_cluster.cc:1366] Running /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/wal
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/logs
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data/info.pb
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.247.189:35611
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.247.189
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.247.189
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.247.148:37647
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.247.190:40563,127.15.247.189:35611,127.15.247.188:40735
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44053/jwks.json
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:30.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:30.951288 16452 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } attempt: 1
03:08:30.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:30.955932 16452 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.189:35611: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111)
03:08:31.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:31.024502 16452 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } attempt: 2
03:08:31.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:31.031548 16452 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.189:35611: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111)
03:08:31.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:31.123144 16452 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } attempt: 3
03:08:31.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:31.129592 16452 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.189:35611: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111)
03:08:31.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:31.273118 16452 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } attempt: 4
03:08:31.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:31.278164 16452 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.189:35611: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111)
03:08:31.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:31.304556 16462 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:31.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:31.305121 16462 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:31.305748 16462 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:31.344202 16462 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:31.344466 16462 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:31.344743 16462 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:31.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:31.344949 16462 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:31.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:31.345219 16462 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:31.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:31.345659 16462 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:31.345947 16462 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44053/jwks.json
03:08:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:31.391268 16462 master_runner.cc:386] Master server non-default flags:
03:08:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.247.148:37647
03:08:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data
03:08:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/wal
03:08:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.247.190:40563,127.15.247.189:35611,127.15.247.188:40735
03:08:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab
03:08:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.247.189
03:08:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.247.189:35611
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44053/jwks.json
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data/info.pb
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.247.189
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/logs
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 68f10dd41067059260a5fb5c725b178b69c38aed
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Mar 2026 02:43:22 UTC on e7f111948823
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11031
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:31.392639 16462 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:31.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:31.393808 16462 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:31.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:31.407289 16469 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:31.577641 16452 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } attempt: 5
03:08:31.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:31.583089 16452 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.189:35611: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111)
03:08:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:32.130604 16452 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } attempt: 6
03:08:32.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:32.164049 16452 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.189:35611: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111)
03:08:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:32.810547 16468 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 16462
03:08:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.170030 16462 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.764s user 0.634s sys 1.095s
03:08:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.170552 16462 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.764s user 0.634s sys 1.095s
03:08:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:31.407598 16470 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:33.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.172672 16471 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1763 milliseconds
03:08:33.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.173182 16472 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:33.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.173758 16462 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:33.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.186184 16462 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:33.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.190214 16462 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:33.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.194089 16462 hybrid_clock.cc:648] HybridClock initialized: now 1774062513193955 us; error 149 us; skew 500 ppm
03:08:33.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.195108 16462 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:33.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:33 dist-test-slave-62xr krb5kdc[16362](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062513, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:33.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.202375 16462 init.cc:377] Logged in from keytab as kudu/127.15.247.189@KRBTEST.COM (short username kudu)
03:08:33.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.208228 16462 webserver.cc:492] Webserver started at http://127.15.247.189:36167/ using document root <none> and password file <none>
03:08:33.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.209401 16462 fs_manager.cc:362] Metadata directory not provided
03:08:33.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.209695 16462 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.210270 16462 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:08:33.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.215111 16462 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data/instance:
03:08:33.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ce0e821f1a84309905e4b5361438a72"
03:08:33.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:33 on dist-test-slave-62xr"
03:08:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.216497 16462 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/wal/instance:
03:08:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ce0e821f1a84309905e4b5361438a72"
03:08:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:33 on dist-test-slave-62xr"
03:08:33.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.224888 16462 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.002s
03:08:33.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.225189 16452 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } attempt: 7
03:08:33.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.233328 16486 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:33.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.233596 16452 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.189:35611: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111)
03:08:33.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.235131 16462 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.001s sys 0.004s
03:08:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.235662 16462 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data,/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/wal
03:08:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ce0e821f1a84309905e4b5361438a72"
03:08:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:33 on dist-test-slave-62xr"
03:08:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.236002 16462 fs_report.cc:389] FS layout report
03:08:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/wal
03:08:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/wal
03:08:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data/data
03:08:33.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:33.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:33.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:33.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:33.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:33.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:33.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:33.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:33.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:33.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:33.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.301955 16462 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:33.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.308286 16462 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:33.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.308749 16462 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:33.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.416010 16462 rpc_server.cc:307] RPC server started. Bound to: 127.15.247.189:35611
03:08:33.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.416081 16537 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.247.189:35611 every 8 connection(s)
03:08:33.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.418792 16462 server_base.cc:1193] Dumped server information to /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data/info.pb
03:08:33.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.423676 16350 external_mini_cluster.cc:1428] Started /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu as pid 16462
03:08:33.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.424201 16350 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/wal/instance
03:08:33.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.427115 16538 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:08:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.247.188@KRBTEST.COM; defaulting to no policy
03:08:33.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.440785 16538 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } has no permanent_uuid. Determining permanent_uuid...
03:08:33.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:08:33.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.247.188@KRBTEST.COM" created.
03:08:33.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:33 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062513, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.189@KRBTEST.COM for kudu/127.15.247.190@KRBTEST.COM
03:08:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:08:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.247.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab.
03:08:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.247.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab.
03:08:33.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.484611 16538 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } has no permanent_uuid. Determining permanent_uuid...
03:08:33.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.15.247.188@KRBTEST.COM; defaulting to no policy
03:08:33.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:08:33.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.15.247.188@KRBTEST.COM" created.
03:08:33.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:33 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062513, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.189@KRBTEST.COM for kudu/127.15.247.189@KRBTEST.COM
03:08:33.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.514741 16538 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } has no permanent_uuid. Determining permanent_uuid...
03:08:33.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.517529 16487 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.247.188:40735: connect: Connection refused (error 111)
03:08:33.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.522284 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.188:40735: Network error: Client connection negotiation failed: client connection to 127.15.247.188:40735: connect: Connection refused (error 111)
03:08:33.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:08:33.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.15.247.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab.
03:08:33.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.15.247.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab.
03:08:33.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.525599 16350 external_mini_cluster.cc:1366] Running /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:33.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:33.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/wal
03:08:33.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data
03:08:33.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:33.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:33.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:33.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/logs
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data/info.pb
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.247.188:40735
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.247.188
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.247.188
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.247.148:37647
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:33.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:33.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.247.190:40563,127.15.247.189:35611,127.15.247.188:40735
03:08:33.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:33.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44053/jwks.json
03:08:33.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:33.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.571931 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } attempt: 1
03:08:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.577131 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.188:40735: Network error: Client connection negotiation failed: client connection to 127.15.247.188:40735: connect: Connection refused (error 111)
03:08:33.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.645722 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } attempt: 2
03:08:33.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.650470 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.188:40735: Network error: Client connection negotiation failed: client connection to 127.15.247.188:40735: connect: Connection refused (error 111)
03:08:33.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.742034 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } attempt: 3
03:08:33.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.747334 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.188:40735: Network error: Client connection negotiation failed: client connection to 127.15.247.188:40735: connect: Connection refused (error 111)
03:08:33.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.888455 16548 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:33.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.889184 16548 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:33.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.889940 16548 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:33.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.890897 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } attempt: 4
03:08:33.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.895880 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.188:40735: Network error: Client connection negotiation failed: client connection to 127.15.247.188:40735: connect: Connection refused (error 111)
03:08:33.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.926496 16548 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:33.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.926831 16548 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:33.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.927268 16548 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:33.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.927690 16548 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:33.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.928004 16548 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:33.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.928325 16548 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:33.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.928555 16548 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44053/jwks.json
03:08:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.968987 16548 master_runner.cc:386] Master server non-default flags:
03:08:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.247.148:37647
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/wal
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.247.190:40563,127.15.247.189:35611,127.15.247.188:40735
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.247.188
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.247.188:40735
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44053/jwks.json
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data/info.pb
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.247.188
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/logs
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 68f10dd41067059260a5fb5c725b178b69c38aed
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Mar 2026 02:43:22 UTC on e7f111948823
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11031
03:08:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:33.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.971238 16548 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:33.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.972746 16548 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:33.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.986474 16554 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:34.195374 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } attempt: 5
03:08:34.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:34.202033 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.188:40735: Network error: Client connection negotiation failed: client connection to 127.15.247.188:40735: connect: Connection refused (error 111)
03:08:34.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:34.749961 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } attempt: 6
03:08:34.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:34.758217 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.188:40735: Network error: Client connection negotiation failed: client connection to 127.15.247.188:40735: connect: Connection refused (error 111)
03:08:35.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.324419 16452 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } attempt: 8
03:08:35.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:35 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062510, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.190@KRBTEST.COM for kudu/127.15.247.189@KRBTEST.COM
03:08:35.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.364902 16452 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } has no permanent_uuid. Determining permanent_uuid...
03:08:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:35.373641 16452 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.188:40735: Network error: Client connection negotiation failed: client connection to 127.15.247.188:40735: connect: Connection refused (error 111)
03:08:35.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.440429 16452 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } attempt: 1
03:08:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:35.445246 16452 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.188:40735: Network error: Client connection negotiation failed: client connection to 127.15.247.188:40735: connect: Connection refused (error 111)
03:08:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.989914 16556 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:33.997320 16558 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:33.994477 16548 server_base.cc:1061] running on GCE node
03:08:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.498183 16548 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.498843 16452 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } attempt: 2
03:08:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:35.504565 16452 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.188:40735: Network error: Client connection negotiation failed: client connection to 127.15.247.188:40735: connect: Connection refused (error 111)
03:08:35.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.508100 16548 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:35.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.509644 16548 hybrid_clock.cc:648] HybridClock initialized: now 1774062515509560 us; error 80 us; skew 500 ppm
03:08:35.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.510564 16548 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:35 dist-test-slave-62xr krb5kdc[16362](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062515, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.521070 16548 init.cc:377] Logged in from keytab as kudu/127.15.247.188@KRBTEST.COM (short username kudu)
03:08:35.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.532891 16548 webserver.cc:492] Webserver started at http://127.15.247.188:35457/ using document root <none> and password file <none>
03:08:35.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.534691 16548 fs_manager.cc:362] Metadata directory not provided
03:08:35.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.535043 16548 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:35.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.535946 16548 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:08:35.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.544633 16548 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data/instance:
03:08:35.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "291fd5213c214c2597138e20d7b79ae3"
03:08:35.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:35 on dist-test-slave-62xr"
03:08:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.546515 16548 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/wal/instance:
03:08:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "291fd5213c214c2597138e20d7b79ae3"
03:08:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:35 on dist-test-slave-62xr"
03:08:35.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.561245 16548 fs_manager.cc:696] Time spent creating directory manager: real 0.014s user 0.005s sys 0.004s
03:08:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.581176 16452 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } attempt: 3
03:08:35.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:35.590566 16452 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.188:40735: Network error: Client connection negotiation failed: client connection to 127.15.247.188:40735: connect: Connection refused (error 111)
03:08:35.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.592011 16572 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.005s user 0.000s sys 0.000s
03:08:35.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.601658 16548 fs_manager.cc:730] Time spent opening block manager: real 0.034s user 0.012s sys 0.008s
03:08:35.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.602185 16548 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data,/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/wal
03:08:35.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "291fd5213c214c2597138e20d7b79ae3"
03:08:35.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:35 on dist-test-slave-62xr"
03:08:35.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.603070 16548 fs_report.cc:389] FS layout report
03:08:35.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:35.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/wal
03:08:35.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/wal
03:08:35.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data/data
03:08:35.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:35.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:35.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:35.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:35.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:35.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:35.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:35.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:35.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:35.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:35.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.680972 16548 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:35.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.688172 16548 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:35.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.688827 16548 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.746125 16452 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } attempt: 4
03:08:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:35.751793 16452 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.247.188:40735: Network error: Client connection negotiation failed: client connection to 127.15.247.188:40735: connect: Connection refused (error 111)
03:08:35.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.784760 16548 rpc_server.cc:307] RPC server started. Bound to: 127.15.247.188:40735
03:08:35.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.784845 16623 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.247.188:40735 every 8 connection(s)
03:08:35.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.788654 16548 server_base.cc:1193] Dumped server information to /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data/info.pb
03:08:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.796075 16350 external_mini_cluster.cc:1428] Started /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu as pid 16548
03:08:35.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.797207 16350 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/wal/instance
03:08:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.797977 16624 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:08:35.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.818693 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } attempt: 7
03:08:35.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.817561 16624 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } has no permanent_uuid. Determining permanent_uuid...
03:08:35.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:35 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062513, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.189@KRBTEST.COM for kudu/127.15.247.188@KRBTEST.COM
03:08:35.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:35 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062507, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.247.188@KRBTEST.COM
03:08:35.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:35 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062515, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.188@KRBTEST.COM for kudu/127.15.247.190@KRBTEST.COM
03:08:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.899188 16624 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } has no permanent_uuid. Determining permanent_uuid...
03:08:35.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.904841 16538 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72: Bootstrap starting.
03:08:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.915525 16538 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72: Neither blocks nor log segments found. Creating new log.
03:08:35.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.918020 16538 log.cc:826] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72: Log is configured to *not* fsync() on all Append() calls
03:08:35.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:35 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062507, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.247.189@KRBTEST.COM
03:08:35.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.929823 16538 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72: No bootstrap required, opened a new log
03:08:35.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:35 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062515, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.188@KRBTEST.COM for kudu/127.15.247.189@KRBTEST.COM
03:08:35.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.951478 16624 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } has no permanent_uuid. Determining permanent_uuid...
03:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:35 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062507, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.247.190@KRBTEST.COM
03:08:35.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:35 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062515, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.188@KRBTEST.COM for kudu/127.15.247.188@KRBTEST.COM
03:08:35.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.972100 16538 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:35.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.972892 16538 raft_consensus.cc:385] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:08:35.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.973225 16538 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ce0e821f1a84309905e4b5361438a72, State: Initialized, Role: FOLLOWER
03:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.974370 16538 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.979405 16635 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:35.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.980775 16635 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [sys.catalog]: This master's current role is: FOLLOWER
03:08:35.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.981797 16538 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [sys.catalog]: configured and running, proceeding with master startup.
03:08:35.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:35.998091 16624 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: Bootstrap starting.
03:08:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.001652 16350 external_mini_cluster.cc:949] 0 TS(s) registered with all masters
03:08:36.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.008620 16624 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: Neither blocks nor log segments found. Creating new log.
03:08:36.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.012579 16624 log.cc:826] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: Log is configured to *not* fsync() on all Append() calls
03:08:36.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.019308 16624 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: No bootstrap required, opened a new log
03:08:36.029 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:36.030 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
03:08:36.032 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
03:08:36.032 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
03:08:36.034 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.15.247.190"
port: 40563
}
bound_http_address {
host: "127.15.247.190"
port: 45525
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.15.247.189"
port: 35611
}
bound_http_address {
host: "127.15.247.189"
port: 36167
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.15.247.188"
port: 40735
}
bound_http_address {
host: "127.15.247.188"
port: 35457
}
}
}
03:08:36.034 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:08:36.035 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:36.035 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:08:36.035 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:08:36.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.048483 16452 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } attempt: 5
03:08:36.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:36.057300 16647 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:08:36.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:36.058228 16647 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:08:36.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:36.071875 16492 tablet.cc:2379] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:08:36.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:36 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062510, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.190@KRBTEST.COM for kudu/127.15.247.188@KRBTEST.COM
03:08:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.077577 16624 raft_consensus.cc:359] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.079694 16624 raft_consensus.cc:385] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:08:36.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:36 dist-test-slave-62xr krb5kdc[16362](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062516, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:36.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.080086 16624 raft_consensus.cc:740] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 291fd5213c214c2597138e20d7b79ae3, State: Initialized, Role: FOLLOWER
03:08:36.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.034106 16462 master_runner.cc:186] Error getting master registration for 127.15.247.190:40563: OK, instance_id { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" instance_seqno: 1774062510798259 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:08:36.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.098332 16462 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:08:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.099270 16624 consensus_queue.cc:260] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:36.115 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:36.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.115449 16624 sys_catalog.cc:565] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: configured and running, proceeding with master startup.
03:08:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.119906 16651 sys_catalog.cc:455] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.120925 16651 sys_catalog.cc:458] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: This master's current role is: FOLLOWER
03:08:36.124 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5cc
03:08:36.127 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:08:36.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.130916 16452 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: Bootstrap starting.
03:08:36.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.143211 16452 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: Neither blocks nor log segments found. Creating new log.
03:08:36.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.146070 16452 log.cc:826] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: Log is configured to *not* fsync() on all Append() calls
03:08:36.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.159214 16452 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: No bootstrap required, opened a new log
03:08:36.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:36.196345 16578 tablet.cc:2379] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:08:36.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:36.197748 16663 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:08:36.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:36.198127 16663 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:08:36.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.207211 16651 raft_consensus.cc:493] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:08:36.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.207906 16651 raft_consensus.cc:515] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.211117 16452 raft_consensus.cc:359] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:36.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.213205 16452 raft_consensus.cc:385] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:08:36.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.213835 16452 raft_consensus.cc:740] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d5c213d669ed47ed81f884b52df3c0b6, State: Initialized, Role: FOLLOWER
03:08:36.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.215392 16513 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "291fd5213c214c2597138e20d7b79ae3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2ce0e821f1a84309905e4b5361438a72" is_pre_election: true
03:08:36.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.215775 16452 consensus_queue.cc:260] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:36.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.216403 16513 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 291fd5213c214c2597138e20d7b79ae3 in term 0.
03:08:36.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.220139 16573 leader_election.cc:304] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 291fd5213c214c2597138e20d7b79ae3, 2ce0e821f1a84309905e4b5361438a72; no voters:
03:08:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.222579 16651 leader_election.cc:290] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d5c213d669ed47ed81f884b52df3c0b6 (127.15.247.190:40563), 2ce0e821f1a84309905e4b5361438a72 (127.15.247.189:35611)
03:08:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.223446 16651 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:08:36.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.224102 16651 raft_consensus.cc:493] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:08:36.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.224740 16651 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 0 FOLLOWER]: Advancing to term 1
03:08:36.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.168694 16548 master_runner.cc:186] Error getting master registration for 127.15.247.190:40563: OK, instance_id { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" instance_seqno: 1774062510798259 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:08:36.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.237121 16548 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:36.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.239531 16426 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "291fd5213c214c2597138e20d7b79ae3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d5c213d669ed47ed81f884b52df3c0b6" is_pre_election: true
03:08:36.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.241588 16426 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 291fd5213c214c2597138e20d7b79ae3 in term 0.
03:08:36.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.241385 16665 sys_catalog.cc:455] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:36.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.236272 16452 sys_catalog.cc:565] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: configured and running, proceeding with master startup.
03:08:36.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.243088 16651 raft_consensus.cc:515] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:36.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.245781 16665 sys_catalog.cc:458] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: This master's current role is: FOLLOWER
03:08:36.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.250185 16426 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "291fd5213c214c2597138e20d7b79ae3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d5c213d669ed47ed81f884b52df3c0b6"
03:08:36.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:36.251495 16405 tablet.cc:2379] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:08:36.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.251626 16513 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "291fd5213c214c2597138e20d7b79ae3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2ce0e821f1a84309905e4b5361438a72"
03:08:36.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.251740 16426 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 0 FOLLOWER]: Advancing to term 1
03:08:36.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.252633 16513 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [term 0 FOLLOWER]: Advancing to term 1
03:08:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.262436 16426 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 291fd5213c214c2597138e20d7b79ae3 in term 1.
03:08:36.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.263068 16513 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 291fd5213c214c2597138e20d7b79ae3 in term 1.
03:08:36.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.264642 16573 leader_election.cc:304] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 291fd5213c214c2597138e20d7b79ae3, 2ce0e821f1a84309905e4b5361438a72; no voters:
03:08:36.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.265506 16651 leader_election.cc:290] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [CANDIDATE]: Term 1 election: Requested vote from peers d5c213d669ed47ed81f884b52df3c0b6 (127.15.247.190:40563), 2ce0e821f1a84309905e4b5361438a72 (127.15.247.189:35611)
03:08:36.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.265836 16666 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 1 FOLLOWER]: Leader election won for term 1
03:08:36.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.266438 16666 raft_consensus.cc:697] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 1 LEADER]: Becoming Leader. State: Replica: 291fd5213c214c2597138e20d7b79ae3, State: Running, Role: LEADER
03:08:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.267563 16666 consensus_queue.cc:237] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:36.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.290127 16651 sys_catalog.cc:455] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 291fd5213c214c2597138e20d7b79ae3. Latest consensus state: current_term: 1 leader_uuid: "291fd5213c214c2597138e20d7b79ae3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:36.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.291252 16651 sys_catalog.cc:458] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: This master's current role is: LEADER
03:08:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.300612 16671 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:08:36.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.311918 16671 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:08:36.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.350489 16513 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [term 1 FOLLOWER]: Refusing update from remote peer 291fd5213c214c2597138e20d7b79ae3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:08:36.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.355765 16651 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
03:08:36.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.366788 16426 raft_consensus.cc:1275] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 1 FOLLOWER]: Refusing update from remote peer 291fd5213c214c2597138e20d7b79ae3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:08:36.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.371016 16651 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
03:08:36.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.406554 16635 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 291fd5213c214c2597138e20d7b79ae3. Latest consensus state: current_term: 1 leader_uuid: "291fd5213c214c2597138e20d7b79ae3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:36.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.409327 16635 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [sys.catalog]: This master's current role is: FOLLOWER
03:08:36.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.420967 16675 mvcc.cc:204] Tried to move back new op lower bound from 7266560066933870592 to 7266560066651021312. Current Snapshot: MvccSnapshot[applied={T|T < 7266560066933870592}]
03:08:36.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.428427 16666 sys_catalog.cc:455] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "291fd5213c214c2597138e20d7b79ae3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.429894 16666 sys_catalog.cc:458] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: This master's current role is: LEADER
03:08:36.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.434122 16671 catalog_manager.cc:1357] Generated new cluster ID: fa55dc7aa4c746a9a387288c81e3ed4e
03:08:36.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.434814 16671 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:08:36.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.439735 16665 sys_catalog.cc:455] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 291fd5213c214c2597138e20d7b79ae3. Latest consensus state: current_term: 1 leader_uuid: "291fd5213c214c2597138e20d7b79ae3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:36.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.443830 16635 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "291fd5213c214c2597138e20d7b79ae3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:36.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.445833 16635 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [sys.catalog]: This master's current role is: FOLLOWER
03:08:36.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.457549 16665 sys_catalog.cc:458] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: This master's current role is: FOLLOWER
03:08:36.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.473599 16666 sys_catalog.cc:455] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "291fd5213c214c2597138e20d7b79ae3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:36.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.475020 16666 sys_catalog.cc:458] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: This master's current role is: LEADER
03:08:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.480405 16680 mvcc.cc:204] Tried to move back new op lower bound from 7266560066933870592 to 7266560066651021312. Current Snapshot: MvccSnapshot[applied={T|T < 7266560066933870592}]
03:08:36.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.487159 16687 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:08:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.499010 16665 sys_catalog.cc:455] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "291fd5213c214c2597138e20d7b79ae3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.499871 16665 sys_catalog.cc:458] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: This master's current role is: FOLLOWER
03:08:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.573022 16671 catalog_manager.cc:1380] Generated new certificate authority record
03:08:36.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.576467 16671 catalog_manager.cc:1514] Loading token signing keys...
03:08:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:36.618182 16335 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad800 after lost signal to thread 16210
03:08:36.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:36.619467 16335 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6220 after lost signal to thread 16338
03:08:36.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.640659 16671 catalog_manager.cc:6044] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: Generated new TSK 0
03:08:36.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:36.641937 16671 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:08:37.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:37.064911 16647 catalog_manager.cc:1269] Loaded cluster ID: fa55dc7aa4c746a9a387288c81e3ed4e
03:08:37.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:37.065541 16647 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72: loading cluster ID for follower catalog manager: success
03:08:37.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:37.084362 16647 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72: acquiring CA information for follower catalog manager: success
03:08:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:37.099092 16647 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:37.132 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.247.190:40563
03:08:37.132 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:08:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:37.133219 16350 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu with pid 16379
03:08:37.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:37.174497 16576 connection.cc:537] client connection to 127.15.247.190:40563 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:08:37.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:37.174983 16576 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:08:37.191 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:37.191 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.247.188:40735
03:08:37.192 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:08:37.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:37.192569 16350 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu with pid 16548
03:08:37.236 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:37.236 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.247.189:35611
03:08:37.236 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:08:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:37.236969 16350 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu with pid 16462
03:08:37.273 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:37.273 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.247.190:40563
03:08:37.274 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:08:37.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:37.276160 16350 external_mini_cluster.cc:1366] Running /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:37.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:37.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/wal
03:08:37.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data
03:08:37.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:37.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:37.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:37.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:37.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:37.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:37.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/logs
03:08:37.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data/info.pb
03:08:37.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:37.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.247.190:40563
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.247.190
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45525
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.247.190
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.247.148:37647
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.247.190:40563,127.15.247.189:35611,127.15.247.188:40735
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44053/jwks.json
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:37.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:37.716830 16696 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:37.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:37.717582 16696 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:37.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:37.718354 16696 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:37.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:37.757980 16696 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:37.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:37.758427 16696 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:37.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:37.758936 16696 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:37.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:37.759271 16696 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:37.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:37.759629 16696 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:37.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:37.759944 16696 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:37.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:37.760188 16696 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44053/jwks.json
03:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:37.817859 16696 master_runner.cc:386] Master server non-default flags:
03:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.247.148:37647
03:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data
03:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/wal
03:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.247.190:40563,127.15.247.189:35611,127.15.247.188:40735
03:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab
03:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.247.190
03:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.247.190:40563
03:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44053/jwks.json
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data/info.pb
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.247.190
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45525
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/logs
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 68f10dd41067059260a5fb5c725b178b69c38aed
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Mar 2026 02:43:22 UTC on e7f111948823
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11031
03:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:37.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:37.819952 16696 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:37.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:37.821398 16696 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:37.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:37.841586 16702 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:40.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:39.240829 16701 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 16696
03:08:40.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:40.146948 16701 kernel_stack_watchdog.cc:198] Thread 16696 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
03:08:40.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:08:40.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:08:40.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:08:40.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:08:40.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:37.843000 16703 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:40.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:40.149700 16696 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.312s user 0.679s sys 1.234s
03:08:40.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:40.150444 16696 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.313s user 0.680s sys 1.234s
03:08:40.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:40.152480 16705 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:40.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:40.168406 16704 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2328 milliseconds
03:08:40.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.168922 16696 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:40.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.172501 16696 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:40.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.176756 16696 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:40.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.179193 16696 hybrid_clock.cc:648] HybridClock initialized: now 1774062520179147 us; error 54 us; skew 500 ppm
03:08:40.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.181075 16696 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:40.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:40 dist-test-slave-62xr krb5kdc[16362](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062520, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:40.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.195212 16696 init.cc:377] Logged in from keytab as kudu/127.15.247.190@KRBTEST.COM (short username kudu)
03:08:40.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.205506 16696 webserver.cc:492] Webserver started at http://127.15.247.190:45525/ using document root <none> and password file <none>
03:08:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.206820 16696 fs_manager.cc:362] Metadata directory not provided
03:08:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.207067 16696 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:40.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.221318 16696 fs_manager.cc:714] Time spent opening directory manager: real 0.011s user 0.010s sys 0.000s
03:08:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.231056 16715 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:40.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.234338 16696 fs_manager.cc:730] Time spent opening block manager: real 0.010s user 0.005s sys 0.001s
03:08:40.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.234815 16696 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data,/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/wal
03:08:40.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d5c213d669ed47ed81f884b52df3c0b6"
03:08:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:30 on dist-test-slave-62xr"
03:08:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.238936 16696 fs_report.cc:389] FS layout report
03:08:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/wal
03:08:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/wal
03:08:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data/data
03:08:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.323804 16696 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:40.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.332357 16696 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.332962 16696 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.441911 16696 rpc_server.cc:307] RPC server started. Bound to: 127.15.247.190:40563
03:08:40.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.442415 16767 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.247.190:40563 every 8 connection(s)
03:08:40.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.446260 16696 server_base.cc:1193] Dumped server information to /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data/info.pb
03:08:40.452 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:40.452 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.247.188:40735
03:08:40.452 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:08:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.450512 16350 external_mini_cluster.cc:1428] Started /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu as pid 16696
03:08:40.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.455310 16350 external_mini_cluster.cc:1366] Running /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:40.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:40.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/wal
03:08:40.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data
03:08:40.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:40.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:40.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:40.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:40.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:40.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:40.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/logs
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data/info.pb
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.247.188:40735
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.247.188
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35457
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.247.188
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.247.148:37647
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.247.190:40563,127.15.247.189:35611,127.15.247.188:40735
03:08:40.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:40.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44053/jwks.json
03:08:40.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:40.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:40.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.470286 16768 sys_catalog.cc:263] Verifying existing consensus state
03:08:40.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.484175 16768 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: Bootstrap starting.
03:08:40.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.537890 16768 log.cc:826] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: Log is configured to *not* fsync() on all Append() calls
03:08:40.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.566119 16768 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:40.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.567124 16768 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: Bootstrap complete.
03:08:40.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.601167 16768 raft_consensus.cc:359] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:40.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.604774 16768 raft_consensus.cc:740] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: d5c213d669ed47ed81f884b52df3c0b6, State: Initialized, Role: FOLLOWER
03:08:40.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.606649 16768 consensus_queue.cc:260] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:40.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.611618 16772 sys_catalog.cc:455] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:40.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.614025 16772 sys_catalog.cc:458] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: This master's current role is: FOLLOWER
03:08:40.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.615911 16768 sys_catalog.cc:565] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: configured and running, proceeding with master startup.
03:08:40.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.675673 16783 catalog_manager.cc:1269] Loaded cluster ID: fa55dc7aa4c746a9a387288c81e3ed4e
03:08:40.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.675989 16783 catalog_manager.cc:1562] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: loading cluster ID for follower catalog manager: success
03:08:40.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.684682 16783 catalog_manager.cc:1584] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: acquiring CA information for follower catalog manager: success
03:08:40.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:40 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062520, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.190@KRBTEST.COM for kudu/127.15.247.190@KRBTEST.COM
03:08:40.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.698364 16783 catalog_manager.cc:1612] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:40.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.734273 16696 master_runner.cc:186] Error getting master registration for 127.15.247.189:35611: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111),
03:08:40.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:40.779476 16696 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:40.973062 16770 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:40.973707 16770 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:40.974458 16770 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:41.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:41.013351 16770 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:41.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:41.013715 16770 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:41.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:41.014001 16770 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:41.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:41.014516 16770 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:41.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:41.014784 16770 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:41.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:41.015057 16770 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:41.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:41.015225 16770 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44053/jwks.json
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:41.076637 16770 master_runner.cc:386] Master server non-default flags:
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.247.148:37647
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/wal
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.247.190:40563,127.15.247.189:35611,127.15.247.188:40735
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.247.188
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.247.188:40735
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44053/jwks.json
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data/info.pb
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.247.188
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35457
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/logs
03:08:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 68f10dd41067059260a5fb5c725b178b69c38aed
03:08:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Mar 2026 02:43:22 UTC on e7f111948823
03:08:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11031
03:08:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:41.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:41.078459 16770 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:41.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:41.079838 16770 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:41.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:41.102221 16793 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:41.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:41.199242 16335 debug-util.cc:398] Leaking SignalData structure 0x7b08000b60a0 after lost signal to thread 16210
03:08:41.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:41.200964 16335 debug-util.cc:398] Leaking SignalData structure 0x7b080003d1e0 after lost signal to thread 16338
03:08:41.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:41.948386 16800 raft_consensus.cc:493] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:08:41.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:41.949337 16800 raft_consensus.cc:515] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:41.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:41.962463 16800 leader_election.cc:290] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 2ce0e821f1a84309905e4b5361438a72 (127.15.247.189:35611), 291fd5213c214c2597138e20d7b79ae3 (127.15.247.188:40735)
03:08:41.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:41.968861 16717 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111)
03:08:41.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:41.979835 16717 leader_election.cc:336] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 2ce0e821f1a84309905e4b5361438a72 (127.15.247.189:35611): Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111)
03:08:41.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:41.983079 16717 leader_election.cc:336] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 291fd5213c214c2597138e20d7b79ae3 (127.15.247.188:40735): Network error: Client connection negotiation failed: client connection to 127.15.247.188:40735: connect: Connection refused (error 111)
03:08:41.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:41.983585 16717 leader_election.cc:304] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d5c213d669ed47ed81f884b52df3c0b6; no voters: 291fd5213c214c2597138e20d7b79ae3, 2ce0e821f1a84309905e4b5361438a72
03:08:41.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:41.984416 16800 raft_consensus.cc:2749] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
03:08:42.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:41.110695 16796 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:42.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:41.104226 16794 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:42.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:42.842295 16795 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1734 milliseconds
03:08:42.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:42.842553 16770 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:42.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:42.844549 16770 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:42.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:42.850024 16770 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:42.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:42.851699 16770 hybrid_clock.cc:648] HybridClock initialized: now 1774062522851554 us; error 152 us; skew 500 ppm
03:08:42.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:42.852881 16770 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:42.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:42 dist-test-slave-62xr krb5kdc[16362](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062522, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:42.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:42.860896 16770 init.cc:377] Logged in from keytab as kudu/127.15.247.188@KRBTEST.COM (short username kudu)
03:08:42.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:42.867725 16770 webserver.cc:492] Webserver started at http://127.15.247.188:35457/ using document root <none> and password file <none>
03:08:42.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:42.869098 16770 fs_manager.cc:362] Metadata directory not provided
03:08:42.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:42.869441 16770 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:42.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:42.883869 16770 fs_manager.cc:714] Time spent opening directory manager: real 0.009s user 0.006s sys 0.005s
03:08:42.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:42.889999 16809 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:42.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:42.891559 16770 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.004s
03:08:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:42.891906 16770 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data,/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/wal
03:08:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "291fd5213c214c2597138e20d7b79ae3"
03:08:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:35 on dist-test-slave-62xr"
03:08:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:42.894788 16770 fs_report.cc:389] FS layout report
03:08:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/wal
03:08:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/wal
03:08:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data/data
03:08:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:42.972277 16770 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:42.984407 16770 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:42.984927 16770 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.105322 16770 rpc_server.cc:307] RPC server started. Bound to: 127.15.247.188:40735
03:08:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.105415 16862 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.247.188:40735 every 8 connection(s)
03:08:43.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.110543 16770 server_base.cc:1193] Dumped server information to /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data/info.pb
03:08:43.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.121484 16350 external_mini_cluster.cc:1428] Started /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu as pid 16770
03:08:43.122 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:43.122 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.247.189:35611
03:08:43.122 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:08:43.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.126107 16350 external_mini_cluster.cc:1366] Running /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/wal
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/logs
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data/info.pb
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.247.189:35611
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.247.189
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36167
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.247.189
03:08:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.247.148:37647
03:08:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.247.190:40563,127.15.247.189:35611,127.15.247.188:40735
03:08:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44053/jwks.json
03:08:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:43.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.136708 16863 sys_catalog.cc:263] Verifying existing consensus state
03:08:43.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.145668 16863 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: Bootstrap starting.
03:08:43.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.209722 16863 log.cc:826] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: Log is configured to *not* fsync() on all Append() calls
03:08:43.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.257412 16863 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:43.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.259093 16863 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: Bootstrap complete.
03:08:43.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.312865 16863 raft_consensus.cc:359] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:43.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.316529 16863 raft_consensus.cc:740] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 291fd5213c214c2597138e20d7b79ae3, State: Initialized, Role: FOLLOWER
03:08:43.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.317925 16863 consensus_queue.cc:260] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:43.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.322803 16867 sys_catalog.cc:455] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:43.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.323901 16863 sys_catalog.cc:565] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: configured and running, proceeding with master startup.
03:08:43.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.324152 16867 sys_catalog.cc:458] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: This master's current role is: FOLLOWER
03:08:43.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.432523 16878 catalog_manager.cc:1269] Loaded cluster ID: fa55dc7aa4c746a9a387288c81e3ed4e
03:08:43.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.432828 16878 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: loading cluster ID for follower catalog manager: success
03:08:43.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.441730 16878 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: acquiring CA information for follower catalog manager: success
03:08:43.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:43 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062522, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.188@KRBTEST.COM for kudu/127.15.247.190@KRBTEST.COM
03:08:43.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.448972 16878 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:43.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.501001 16770 master_runner.cc:186] Error getting master registration for 127.15.247.189:35611: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111),
03:08:43.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.530701 16770 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:43.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:43.679826 16865 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:43.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:43.680725 16865 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:43.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:43.681999 16865 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:43.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:43.742385 16865 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:43.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:43.742846 16865 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:43.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:43.743844 16865 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:43.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:43.744107 16865 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:43.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:43.744304 16865 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:43.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:43.744552 16865 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:43.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:43.744801 16865 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44053/jwks.json
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.801693 16865 master_runner.cc:386] Master server non-default flags:
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.247.148:37647
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/wal
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.247.190:40563,127.15.247.189:35611,127.15.247.188:40735
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.247.189
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.247.189:35611
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44053/jwks.json
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data/info.pb
03:08:43.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.247.189
03:08:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36167
03:08:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/logs
03:08:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 68f10dd41067059260a5fb5c725b178b69c38aed
03:08:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Mar 2026 02:43:22 UTC on e7f111948823
03:08:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11031
03:08:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.803654 16865 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:43.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.804913 16865 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:43.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:43.824350 16888 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:43.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.859112 16892 raft_consensus.cc:493] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:08:43.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.863734 16892 raft_consensus.cc:515] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:43.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.878782 16892 leader_election.cc:290] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 2ce0e821f1a84309905e4b5361438a72 (127.15.247.189:35611), 291fd5213c214c2597138e20d7b79ae3 (127.15.247.188:40735)
03:08:43.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.941120 16837 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d5c213d669ed47ed81f884b52df3c0b6" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "291fd5213c214c2597138e20d7b79ae3" is_pre_election: true
03:08:43.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.942610 16837 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d5c213d669ed47ed81f884b52df3c0b6 in term 1.
03:08:43.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.947468 16717 leader_election.cc:304] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 291fd5213c214c2597138e20d7b79ae3, d5c213d669ed47ed81f884b52df3c0b6; no voters:
03:08:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.949258 16892 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 1 FOLLOWER]: Leader pre-election won for term 2
03:08:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.949812 16892 raft_consensus.cc:493] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:08:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.950281 16892 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 1 FOLLOWER]: Advancing to term 2
03:08:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:43.968312 16717 leader_election.cc:336] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 2ce0e821f1a84309905e4b5361438a72 (127.15.247.189:35611): Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111)
03:08:43.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:43.995548 16892 raft_consensus.cc:515] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:44.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:44.029402 16717 leader_election.cc:336] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 2ce0e821f1a84309905e4b5361438a72 (127.15.247.189:35611): Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111)
03:08:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.060338 16837 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d5c213d669ed47ed81f884b52df3c0b6" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "291fd5213c214c2597138e20d7b79ae3"
03:08:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.061209 16837 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 1 FOLLOWER]: Advancing to term 2
03:08:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.081389 16892 leader_election.cc:290] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 2 election: Requested vote from peers 2ce0e821f1a84309905e4b5361438a72 (127.15.247.189:35611), 291fd5213c214c2597138e20d7b79ae3 (127.15.247.188:40735)
03:08:44.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.110373 16837 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d5c213d669ed47ed81f884b52df3c0b6 in term 2.
03:08:44.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.112883 16717 leader_election.cc:304] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 291fd5213c214c2597138e20d7b79ae3, d5c213d669ed47ed81f884b52df3c0b6; no voters: 2ce0e821f1a84309905e4b5361438a72
03:08:44.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.114744 16892 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 2 FOLLOWER]: Leader election won for term 2
03:08:44.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.128748 16892 raft_consensus.cc:697] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 2 LEADER]: Becoming Leader. State: Replica: d5c213d669ed47ed81f884b52df3c0b6, State: Running, Role: LEADER
03:08:44.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.130775 16892 consensus_queue.cc:237] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:44.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.180058 16898 sys_catalog.cc:455] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d5c213d669ed47ed81f884b52df3c0b6. Latest consensus state: current_term: 2 leader_uuid: "d5c213d669ed47ed81f884b52df3c0b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:44.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.181696 16898 sys_catalog.cc:458] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: This master's current role is: LEADER
03:08:44.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.193116 16900 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:08:44.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.242342 16900 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:08:44.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.263741 16900 catalog_manager.cc:1269] Loaded cluster ID: fa55dc7aa4c746a9a387288c81e3ed4e
03:08:44.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.269246 16900 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:08:44.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.280340 16900 catalog_manager.cc:1514] Loading token signing keys...
03:08:44.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.306064 16900 catalog_manager.cc:6055] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: Loaded TSK: 0
03:08:44.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.312808 16900 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:08:44.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:44.647140 16717 consensus_peers.cc:597] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 -> Peer 2ce0e821f1a84309905e4b5361438a72 (127.15.247.189:35611): Couldn't send request to peer 2ce0e821f1a84309905e4b5361438a72. Status: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:08:44.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.752002 16837 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 2 FOLLOWER]: Refusing update from remote peer d5c213d669ed47ed81f884b52df3c0b6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
03:08:44.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.755535 16898 consensus_queue.cc:1048] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
03:08:44.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.797207 16903 sys_catalog.cc:455] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d5c213d669ed47ed81f884b52df3c0b6. Latest consensus state: current_term: 2 leader_uuid: "d5c213d669ed47ed81f884b52df3c0b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:44.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.806941 16903 sys_catalog.cc:458] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: This master's current role is: FOLLOWER
03:08:44.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.827359 16903 sys_catalog.cc:455] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "d5c213d669ed47ed81f884b52df3c0b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:44.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.828348 16903 sys_catalog.cc:458] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: This master's current role is: FOLLOWER
03:08:44.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.831432 16898 sys_catalog.cc:455] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "d5c213d669ed47ed81f884b52df3c0b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:44.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:44.832576 16898 sys_catalog.cc:458] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: This master's current role is: LEADER
03:08:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:45.789650 16696 master_runner.cc:186] Error getting master registration for 127.15.247.189:35611: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111),
03:08:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:45.790449 16696 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:46.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:43.831650 16889 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:46.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:45.283926 16887 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 16865
03:08:46.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:46.402302 16865 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.571s user 0.886s sys 1.388s
03:08:46.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:46.404616 16865 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.574s user 0.887s sys 1.388s
03:08:46.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:46.406085 16891 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:46.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.421407 16865 server_base.cc:1061] running on GCE node
03:08:46.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.423437 16865 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.430032 16865 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.434957 16865 hybrid_clock.cc:648] HybridClock initialized: now 1774062526435229 us; error 746 us; skew 500 ppm
03:08:46.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.437714 16865 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:46.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:46 dist-test-slave-62xr krb5kdc[16362](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062526, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:46.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.454646 16865 init.cc:377] Logged in from keytab as kudu/127.15.247.189@KRBTEST.COM (short username kudu)
03:08:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.466162 16865 webserver.cc:492] Webserver started at http://127.15.247.189:36167/ using document root <none> and password file <none>
03:08:46.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.468331 16865 fs_manager.cc:362] Metadata directory not provided
03:08:46.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.468734 16865 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:46.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.486716 16865 fs_manager.cc:714] Time spent opening directory manager: real 0.010s user 0.004s sys 0.004s
03:08:46.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.493993 16920 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:46.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.496362 16865 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.004s sys 0.002s
03:08:46.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.497329 16865 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data,/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/wal
03:08:46.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ce0e821f1a84309905e4b5361438a72"
03:08:46.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:33 on dist-test-slave-62xr"
03:08:46.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.500030 16865 fs_report.cc:389] FS layout report
03:08:46.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:46.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/wal
03:08:46.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/wal
03:08:46.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data/data
03:08:46.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:46.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:46.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:46.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:46.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:46.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:46.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:46.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:46.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:46.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:46.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:46.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.598627 16865 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:46.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.611858 16865 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:46.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.613677 16865 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:46.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.725301 16865 rpc_server.cc:307] RPC server started. Bound to: 127.15.247.189:35611
03:08:46.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.725428 16972 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.247.189:35611 every 8 connection(s)
03:08:46.738 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:46.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.735457 16865 server_base.cc:1193] Dumped server information to /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data/info.pb
03:08:46.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.737391 16350 external_mini_cluster.cc:1428] Started /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu as pid 16865
03:08:46.742 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.247.190:40563(127.15.247.190:40563)] connecting to peer
03:08:46.755 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.247.188:40735(127.15.247.188:40735)] connecting to peer
03:08:46.761 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.247.189:35611(127.15.247.189:35611)] connecting to peer
03:08:46.769 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.15.247.190:40563(127.15.247.190:40563)] Successfully connected to peer
03:08:46.773 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.15.247.188:40735(127.15.247.188:40735)] Successfully connected to peer
03:08:46.781 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
03:08:46.781 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5cc
03:08:46.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.785466 16973 sys_catalog.cc:263] Verifying existing consensus state
03:08:46.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:46 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1774062516, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:46.789 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:08:46.789 [INFO - kudu-nio-0] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
03:08:46.792 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.15.247.190
03:08:46.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.798401 16973 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72: Bootstrap starting.
03:08:46.809 [DEBUG - kudu-nio-1] (SecurityContext.java:222) Refreshing Kerberos credentials...
03:08:46.809 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5cc
03:08:46.816 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.15.247.189:35611(127.15.247.189:35611)] Successfully connected to peer
03:08:46.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:46 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1774062516, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:46.824 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:08:46.824 [INFO - kudu-nio-1] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
03:08:46.828 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.15.247.188
03:08:46.842 [DEBUG - kudu-nio-2] (SecurityContext.java:222) Refreshing Kerberos credentials...
03:08:46.846 [DEBUG - kudu-nio-2] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5cc
03:08:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:46 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1774062516, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.247.190@KRBTEST.COM
03:08:46.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:46 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1774062516, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:46.880 [DEBUG - kudu-nio-2] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:08:46.880 [INFO - kudu-nio-2] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
03:08:46.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.883805 16973 log.cc:826] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72: Log is configured to *not* fsync() on all Append() calls
03:08:46.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:46 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1774062516, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.247.188@KRBTEST.COM
03:08:46.885 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.15.247.189
03:08:46.908 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x04f72c12, L:/127.0.0.1:58002 - R:/127.15.247.188:40735] using SASL/GSSAPI
03:08:46.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.934978 16973 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:46.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.936154 16973 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72: Bootstrap complete.
03:08:46.944 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xafe05204, L:/127.0.0.1:44326 - R:/127.15.247.190:40563] using SASL/GSSAPI
03:08:46.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:46 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1774062516, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.247.189@KRBTEST.COM
03:08:46.967 [DEBUG - kudu-nio-0] (NetUtil.java:139) Resolved '127.15.247.190' into [/127.15.247.190] in 4664199ns
03:08:46.967 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:08:46.968 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=229), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.247.190:40563, [14ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.247.188:40735, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.247.189:35611, [176ms] Sub RPC ConnectToMaster: received response from server master-127.15.247.188:40735: OK, [215ms] Sub RPC ConnectToMaster: received response from server master-127.15.247.190:40563: OK, deferred=null) after lookup
03:08:46.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.975754 16973 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:46.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.979521 16973 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ce0e821f1a84309905e4b5361438a72, State: Initialized, Role: FOLLOWER
03:08:46.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:46.980778 16973 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:47.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:47.004225 16983 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:47.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:47.007198 16983 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [sys.catalog]: This master's current role is: FOLLOWER
03:08:47.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:47.000094 16973 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [sys.catalog]: configured and running, proceeding with master startup.
03:08:47.104 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xf4a88a4d, L:/127.0.0.1:47604 - R:/127.15.247.189:35611] using SASL/GSSAPI
03:08:47.113 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.15.247.189:35611: Catalog manager is not initialized. State: Starting
03:08:47.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:47 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062520, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.190@KRBTEST.COM for kudu/127.15.247.189@KRBTEST.COM
03:08:47.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:47.231405 16947 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [term 1 FOLLOWER]: Advancing to term 2
03:08:47.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:47.248490 16995 catalog_manager.cc:1269] Loaded cluster ID: fa55dc7aa4c746a9a387288c81e3ed4e
03:08:47.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:47.249075 16995 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72: loading cluster ID for follower catalog manager: success
03:08:47.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:47.258888 16995 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72: acquiring CA information for follower catalog manager: success
03:08:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:47 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062526, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.189@KRBTEST.COM for kudu/127.15.247.190@KRBTEST.COM
03:08:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:47.288276 16995 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:47.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:47.315716 16983 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d5c213d669ed47ed81f884b52df3c0b6. Latest consensus state: current_term: 2 leader_uuid: "d5c213d669ed47ed81f884b52df3c0b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:47.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:47.316671 16983 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [sys.catalog]: This master's current role is: FOLLOWER
03:08:47.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:47.319298 17000 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "d5c213d669ed47ed81f884b52df3c0b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:47.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:47.320161 17000 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72 [sys.catalog]: This master's current role is: FOLLOWER
03:08:47.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:47.371811 15704 debug-util.cc:398] Leaking SignalData structure 0x7b080009f040 after lost signal to thread 15641
03:08:47.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:47.372557 15704 debug-util.cc:398] Leaking SignalData structure 0x7b0800041680 after lost signal to thread 15707
03:08:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:47.623220 16898 sys_catalog.cc:455] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "d5c213d669ed47ed81f884b52df3c0b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:47.624300 16898 sys_catalog.cc:458] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: This master's current role is: LEADER
03:08:48.023 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.247.190:40563
03:08:48.025 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:08:48.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:48.025992 16350 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu with pid 16696
03:08:48.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:48.052034 15779 debug-util.cc:398] Leaking SignalData structure 0x7b08000a7e60 after lost signal to thread 15714
03:08:48.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:48.053553 15779 debug-util.cc:398] Leaking SignalData structure 0x7b080007fdc0 after lost signal to thread 15782
03:08:48.085 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.15.247.190:40563(127.15.247.190:40563)] handling channelInactive
03:08:48.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.247.190:40563(127.15.247.190:40563)] cleaning up while in state READY due to: connection closed
03:08:48.087 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:48.088 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.247.188:40735
03:08:48.088 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:08:48.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:48.088825 16350 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu with pid 16770
03:08:48.157 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.15.247.188:40735(127.15.247.188:40735)] handling channelInactive
03:08:48.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.247.188:40735(127.15.247.188:40735)] cleaning up while in state READY due to: connection closed
03:08:48.160 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:48.161 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.247.189:35611
03:08:48.161 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:08:48.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:48.161479 16350 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu with pid 16865
03:08:48.193 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.15.247.189:35611(127.15.247.189:35611)] handling channelInactive
03:08:48.194 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.15.247.189:35611(127.15.247.189:35611)] cleaning up while in state READY due to: connection closed
03:08:48.194 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:48.195 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.247.190:40563
03:08:48.197 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:48.198660 16350 external_mini_cluster.cc:1366] Running /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/wal
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/logs
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data/info.pb
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.247.190:40563
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.247.190
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45525
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.247.190
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.247.148:37647
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.247.190:40563,127.15.247.189:35611,127.15.247.188:40735
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44053/jwks.json
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:48.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:48.632611 17005 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:48.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:48.633366 17005 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:48.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:48.634371 17005 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:48.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:48.675449 17005 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:48.675851 17005 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:48.676292 17005 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:48.676674 17005 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:48.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:48.676986 17005 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:48.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:48.677323 17005 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:48.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:48.677899 17005 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44053/jwks.json
03:08:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:48.721977 17005 master_runner.cc:386] Master server non-default flags:
03:08:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.247.148:37647
03:08:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data
03:08:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/wal
03:08:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.247.190:40563,127.15.247.189:35611,127.15.247.188:40735
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.247.190
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.247.190:40563
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44053/jwks.json
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data/info.pb
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.247.190
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45525
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/logs
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:48.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:48.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:48.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:48.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 68f10dd41067059260a5fb5c725b178b69c38aed
03:08:48.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:48.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Mar 2026 02:43:22 UTC on e7f111948823
03:08:48.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11031
03:08:48.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:48.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:48.723795 17005 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:48.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:48.725009 17005 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:49.115675 15779 debug-util.cc:398] Leaking SignalData structure 0x7b080003d0a0 after lost signal to thread 15714
03:08:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:49.116619 15779 debug-util.cc:398] Leaking SignalData structure 0x7b08000807a0 after lost signal to thread 15782
03:08:50.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:48.739760 17012 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:50.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:50.138839 17011 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 17005
03:08:50.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:50.638283 17011 kernel_stack_watchdog.cc:198] Thread 17005 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
03:08:50.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:08:50.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:08:50.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:08:50.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:08:50.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:48.739804 17013 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:50.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:50.639706 17005 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.902s user 0.652s sys 1.040s
03:08:50.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:50.640199 17005 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.902s user 0.652s sys 1.040s
03:08:50.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:50.641211 17015 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:50.647246 17014 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1899 milliseconds
03:08:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.647298 17005 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:50.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.649469 17005 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:50.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.653827 17005 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:50.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.655440 17005 hybrid_clock.cc:648] HybridClock initialized: now 1774062530655280 us; error 151 us; skew 500 ppm
03:08:50.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.656888 17005 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:50 dist-test-slave-62xr krb5kdc[16362](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062530, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:50.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.665975 17005 init.cc:377] Logged in from keytab as kudu/127.15.247.190@KRBTEST.COM (short username kudu)
03:08:50.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.673300 17005 webserver.cc:492] Webserver started at http://127.15.247.190:45525/ using document root <none> and password file <none>
03:08:50.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.676071 17005 fs_manager.cc:362] Metadata directory not provided
03:08:50.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.676867 17005 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:50.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.686510 17005 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
03:08:50.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.692461 17025 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:50.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.693840 17005 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
03:08:50.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.694146 17005 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data,/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/wal
03:08:50.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d5c213d669ed47ed81f884b52df3c0b6"
03:08:50.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:30 on dist-test-slave-62xr"
03:08:50.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.696250 17005 fs_report.cc:389] FS layout report
03:08:50.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:50.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/wal
03:08:50.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/wal
03:08:50.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data/data
03:08:50.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:50.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:50.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:50.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:50.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:50.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:50.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:50.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:50.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:50.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:50.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:50.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.767786 17005 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:50.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.777993 17005 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:50.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.778633 17005 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:50.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.876646 17005 rpc_server.cc:307] RPC server started. Bound to: 127.15.247.190:40563
03:08:50.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.876775 17076 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.247.190:40563 every 8 connection(s)
03:08:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.879966 17005 server_base.cc:1193] Dumped server information to /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-0/data/info.pb
03:08:50.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.882501 16350 external_mini_cluster.cc:1428] Started /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu as pid 17005
03:08:50.883 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:50.883 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.247.188:40735
03:08:50.883 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:08:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.887735 16350 external_mini_cluster.cc:1366] Running /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/wal
03:08:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data
03:08:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/logs
03:08:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data/info.pb
03:08:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.247.188:40735
03:08:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.247.188
03:08:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35457
03:08:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab
03:08:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.247.188
03:08:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:50.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:50.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:50.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:50.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.247.148:37647
03:08:50.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:50.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:50.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:50.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:50.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.247.190:40563,127.15.247.189:35611,127.15.247.188:40735
03:08:50.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:50.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44053/jwks.json
03:08:50.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:50.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:50.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.896351 17077 sys_catalog.cc:263] Verifying existing consensus state
03:08:50.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.905056 17077 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: Bootstrap starting.
03:08:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.948984 17077 log.cc:826] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: Log is configured to *not* fsync() on all Append() calls
03:08:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.968324 17077 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:50.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.969523 17077 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: Bootstrap complete.
03:08:50.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.995664 17077 raft_consensus.cc:359] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:50.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.998857 17077 raft_consensus.cc:740] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: d5c213d669ed47ed81f884b52df3c0b6, State: Initialized, Role: FOLLOWER
03:08:51.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:50.999847 17077 consensus_queue.cc:260] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:51.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:51.003087 17081 sys_catalog.cc:455] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:51.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:51.004400 17081 sys_catalog.cc:458] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: This master's current role is: FOLLOWER
03:08:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:51.005631 17077 sys_catalog.cc:565] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: configured and running, proceeding with master startup.
03:08:51.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:51.057624 17094 catalog_manager.cc:1269] Loaded cluster ID: fa55dc7aa4c746a9a387288c81e3ed4e
03:08:51.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:51.058250 17094 catalog_manager.cc:1562] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: loading cluster ID for follower catalog manager: success
03:08:51.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:51.067204 17094 catalog_manager.cc:1584] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: acquiring CA information for follower catalog manager: success
03:08:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:51 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062530, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.190@KRBTEST.COM for kudu/127.15.247.190@KRBTEST.COM
03:08:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:51.078146 17094 catalog_manager.cc:1612] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:51.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:51.105360 17005 master_runner.cc:186] Error getting master registration for 127.15.247.189:35611: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111),
03:08:51.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:51.135434 17005 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:51.321658 17079 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:51.322381 17079 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:51.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:51.323297 17079 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:51.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:51.362677 17079 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:51.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:51.363006 17079 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:51.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:51.363484 17079 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:51.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:51.363813 17079 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:51.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:51.364122 17079 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:51.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:51.364416 17079 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:51.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:51.364658 17079 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44053/jwks.json
03:08:51.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:51.411067 17079 master_runner.cc:386] Master server non-default flags:
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.247.148:37647
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/wal
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.247.190:40563,127.15.247.189:35611,127.15.247.188:40735
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.247.188
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.247.188:40735
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44053/jwks.json
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data/info.pb
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.247.188
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35457
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/logs
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 68f10dd41067059260a5fb5c725b178b69c38aed
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Mar 2026 02:43:22 UTC on e7f111948823
03:08:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11031
03:08:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:51.413023 17079 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:51.414306 17079 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:51.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:51.429098 17103 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:52.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:52.308003 17109 raft_consensus.cc:493] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:08:52.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:52.308979 17109 raft_consensus.cc:515] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:52.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:52.322492 17109 leader_election.cc:290] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 2ce0e821f1a84309905e4b5361438a72 (127.15.247.189:35611), 291fd5213c214c2597138e20d7b79ae3 (127.15.247.188:40735)
03:08:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:52.340658 17026 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111)
03:08:52.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:52.364075 17026 leader_election.cc:336] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 2ce0e821f1a84309905e4b5361438a72 (127.15.247.189:35611): Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111)
03:08:52.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:52.370337 17026 leader_election.cc:336] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 291fd5213c214c2597138e20d7b79ae3 (127.15.247.188:40735): Network error: Client connection negotiation failed: client connection to 127.15.247.188:40735: connect: Connection refused (error 111)
03:08:52.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:52.371397 17026 leader_election.cc:304] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d5c213d669ed47ed81f884b52df3c0b6; no voters: 291fd5213c214c2597138e20d7b79ae3, 2ce0e821f1a84309905e4b5361438a72
03:08:52.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:52.373453 17109 raft_consensus.cc:2749] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
03:08:53.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:51.431504 17104 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:53.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:53.174352 17106 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:53.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:53.179132 17105 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1746 milliseconds
03:08:53.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.179250 17079 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:53.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.182231 17079 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:53.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.186216 17079 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:53.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.187743 17079 hybrid_clock.cc:648] HybridClock initialized: now 1774062533187521 us; error 216 us; skew 500 ppm
03:08:53.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.189046 17079 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:53.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:53 dist-test-slave-62xr krb5kdc[16362](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062533, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:53.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.198000 17079 init.cc:377] Logged in from keytab as kudu/127.15.247.188@KRBTEST.COM (short username kudu)
03:08:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.205322 17079 webserver.cc:492] Webserver started at http://127.15.247.188:35457/ using document root <none> and password file <none>
03:08:53.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.206524 17079 fs_manager.cc:362] Metadata directory not provided
03:08:53.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.206928 17079 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:53.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.220818 17079 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.003s sys 0.005s
03:08:53.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.227283 17120 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:53.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.228870 17079 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
03:08:53.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.229295 17079 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data,/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/wal
03:08:53.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "291fd5213c214c2597138e20d7b79ae3"
03:08:53.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:35 on dist-test-slave-62xr"
03:08:53.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.232100 17079 fs_report.cc:389] FS layout report
03:08:53.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:53.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/wal
03:08:53.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/wal
03:08:53.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data/data
03:08:53.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:53.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:53.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:53.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:53.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:53.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:53.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:53.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:53.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:53.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:53.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:53.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.301525 17079 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:53.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.312536 17079 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:53.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.313511 17079 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:53.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.417640 17079 rpc_server.cc:307] RPC server started. Bound to: 127.15.247.188:40735
03:08:53.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.417699 17171 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.247.188:40735 every 8 connection(s)
03:08:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.422274 17079 server_base.cc:1193] Dumped server information to /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-2/data/info.pb
03:08:53.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.430014 16350 external_mini_cluster.cc:1428] Started /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu as pid 17079
03:08:53.431 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:53.431 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.247.189:35611
03:08:53.431 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.432832 16350 external_mini_cluster.cc:1366] Running /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/wal
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/logs
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data/info.pb
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.247.189:35611
03:08:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.247.189
03:08:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36167
03:08:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab
03:08:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.247.189
03:08:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.247.148:37647
03:08:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.247.190:40563,127.15.247.189:35611,127.15.247.188:40735
03:08:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44053/jwks.json
03:08:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:53.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.446872 17172 sys_catalog.cc:263] Verifying existing consensus state
03:08:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.459074 17172 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: Bootstrap starting.
03:08:53.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.515422 17172 log.cc:826] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: Log is configured to *not* fsync() on all Append() calls
03:08:53.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.540000 17172 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:53.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.541142 17172 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: Bootstrap complete.
03:08:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.584859 17172 raft_consensus.cc:359] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:53.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.589618 17172 raft_consensus.cc:740] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 291fd5213c214c2597138e20d7b79ae3, State: Initialized, Role: FOLLOWER
03:08:53.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.590811 17172 consensus_queue.cc:260] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:53.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.594179 17176 sys_catalog.cc:455] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:53.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.595299 17176 sys_catalog.cc:458] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: This master's current role is: FOLLOWER
03:08:53.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.596724 17172 sys_catalog.cc:565] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: configured and running, proceeding with master startup.
03:08:53.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.640679 17187 catalog_manager.cc:1269] Loaded cluster ID: fa55dc7aa4c746a9a387288c81e3ed4e
03:08:53.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.640998 17187 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: loading cluster ID for follower catalog manager: success
03:08:53.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.648801 17187 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: acquiring CA information for follower catalog manager: success
03:08:53.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.655166 17187 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:53.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:53 dist-test-slave-62xr krb5kdc[16362](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062533, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.188@KRBTEST.COM for kudu/127.15.247.190@KRBTEST.COM
03:08:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.698735 17079 master_runner.cc:186] Error getting master registration for 127.15.247.189:35611: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111),
03:08:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.732436 17079 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:53.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.888058 17193 raft_consensus.cc:493] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:08:53.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.888715 17193 raft_consensus.cc:515] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:53.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.897671 17193 leader_election.cc:290] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 2ce0e821f1a84309905e4b5361438a72 (127.15.247.189:35611), 291fd5213c214c2597138e20d7b79ae3 (127.15.247.188:40735)
03:08:53.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:53.933223 17026 leader_election.cc:336] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 2ce0e821f1a84309905e4b5361438a72 (127.15.247.189:35611): Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111)
03:08:53.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.949383 17147 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d5c213d669ed47ed81f884b52df3c0b6" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "291fd5213c214c2597138e20d7b79ae3" is_pre_election: true
03:08:53.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.950408 17147 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d5c213d669ed47ed81f884b52df3c0b6 in term 2.
03:08:53.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.953552 17026 leader_election.cc:304] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 291fd5213c214c2597138e20d7b79ae3, d5c213d669ed47ed81f884b52df3c0b6; no voters: 2ce0e821f1a84309905e4b5361438a72
03:08:53.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.954952 17193 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 2 FOLLOWER]: Leader pre-election won for term 3
03:08:53.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.955284 17193 raft_consensus.cc:493] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:08:53.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.955960 17193 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 2 FOLLOWER]: Advancing to term 3
03:08:53.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.968993 17193 raft_consensus.cc:515] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:53.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:53.977878 17026 leader_election.cc:336] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 2ce0e821f1a84309905e4b5361438a72 (127.15.247.189:35611): Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111)
03:08:53.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.981452 17147 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d5c213d669ed47ed81f884b52df3c0b6" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "291fd5213c214c2597138e20d7b79ae3"
03:08:53.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.982059 17147 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 2 FOLLOWER]: Advancing to term 3
03:08:53.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.983460 17193 leader_election.cc:290] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 3 election: Requested vote from peers 2ce0e821f1a84309905e4b5361438a72 (127.15.247.189:35611), 291fd5213c214c2597138e20d7b79ae3 (127.15.247.188:40735)
03:08:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:53.991446 17174 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:53.992316 17174 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:53.993459 17174 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.993255 17147 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d5c213d669ed47ed81f884b52df3c0b6 in term 3.
03:08:53.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.995101 17026 leader_election.cc:304] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 291fd5213c214c2597138e20d7b79ae3, d5c213d669ed47ed81f884b52df3c0b6; no voters: 2ce0e821f1a84309905e4b5361438a72
03:08:53.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:53.996752 17193 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 3 FOLLOWER]: Leader election won for term 3
03:08:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.008144 17193 raft_consensus.cc:697] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [term 3 LEADER]: Becoming Leader. State: Replica: d5c213d669ed47ed81f884b52df3c0b6, State: Running, Role: LEADER
03:08:54.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.009769 17193 consensus_queue.cc:237] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } }
03:08:54.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.032106 17200 sys_catalog.cc:455] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d5c213d669ed47ed81f884b52df3c0b6. Latest consensus state: current_term: 3 leader_uuid: "d5c213d669ed47ed81f884b52df3c0b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:54.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.034009 17200 sys_catalog.cc:458] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: This master's current role is: LEADER
03:08:54.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.048971 17202 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:08:54.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.060066 17202 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:08:54.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.064558 17202 catalog_manager.cc:1269] Loaded cluster ID: fa55dc7aa4c746a9a387288c81e3ed4e
03:08:54.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.064891 17202 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:08:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:54.070737 17174 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:54.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.072297 17202 catalog_manager.cc:1514] Loading token signing keys...
03:08:54.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:54.073763 17174 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:54.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:54.074527 17174 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:54.075116 17174 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:54.075639 17174 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:54.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:54.076200 17174 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:54.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:54.076668 17174 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44053/jwks.json
03:08:54.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.077520 17202 catalog_manager.cc:6055] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6: Loaded TSK: 0
03:08:54.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.079500 17202 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:08:54.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.160205 17174 master_runner.cc:386] Master server non-default flags:
03:08:54.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:54.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.247.148:37647
03:08:54.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/wal
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.247.190:40563,127.15.247.189:35611,127.15.247.188:40735
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/kudu.keytab
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.247.189
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.247.189:35611
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44053/jwks.json
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data/info.pb
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.247.189
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36167
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/logs
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 68f10dd41067059260a5fb5c725b178b69c38aed
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Mar 2026 02:43:22 UTC on e7f111948823
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11031
03:08:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.163448 17174 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:54.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.165200 17174 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:54.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:54.194064 17204 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:54.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.545151 17147 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [term 3 FOLLOWER]: Refusing update from remote peer d5c213d669ed47ed81f884b52df3c0b6: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
03:08:54.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.551955 17200 consensus_queue.cc:1048] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.001s
03:08:54.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.590081 17210 sys_catalog.cc:455] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d5c213d669ed47ed81f884b52df3c0b6. Latest consensus state: current_term: 3 leader_uuid: "d5c213d669ed47ed81f884b52df3c0b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:54.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.591549 17210 sys_catalog.cc:458] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: This master's current role is: FOLLOWER
03:08:54.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.640053 17210 sys_catalog.cc:455] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "d5c213d669ed47ed81f884b52df3c0b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:54.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.640976 17210 sys_catalog.cc:458] T 00000000000000000000000000000000 P 291fd5213c214c2597138e20d7b79ae3 [sys.catalog]: This master's current role is: FOLLOWER
03:08:54.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.648618 17200 sys_catalog.cc:455] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "d5c213d669ed47ed81f884b52df3c0b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c213d669ed47ed81f884b52df3c0b6" member_type: VOTER last_known_addr { host: "127.15.247.190" port: 40563 } } peers { permanent_uuid: "2ce0e821f1a84309905e4b5361438a72" member_type: VOTER last_known_addr { host: "127.15.247.189" port: 35611 } } peers { permanent_uuid: "291fd5213c214c2597138e20d7b79ae3" member_type: VOTER last_known_addr { host: "127.15.247.188" port: 40735 } } }
03:08:54.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:54.649755 17200 sys_catalog.cc:458] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 [sys.catalog]: This master's current role is: LEADER
03:08:54.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:54.674360 17026 consensus_peers.cc:597] T 00000000000000000000000000000000 P d5c213d669ed47ed81f884b52df3c0b6 -> Peer 2ce0e821f1a84309905e4b5361438a72 (127.15.247.189:35611): Couldn't send request to peer 2ce0e821f1a84309905e4b5361438a72. Status: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:08:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:55.602741 17203 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 17174
03:08:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:55.984259 17203 kernel_stack_watchdog.cc:198] Thread 17174 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
03:08:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:08:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:08:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:08:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:08:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:54.203958 17205 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:55.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:55.989068 17174 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.794s user 0.689s sys 1.028s
03:08:55.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:55.989490 17174 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.794s user 0.689s sys 1.028s
03:08:55.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:55.990036 17174 server_base.cc:1061] running on GCE node
03:08:55.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:55.991776 17207 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:55.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:55.993430 17174 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:55.997967 17174 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:55.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:55.999536 17174 hybrid_clock.cc:648] HybridClock initialized: now 1774062535999485 us; error 131 us; skew 500 ppm
03:08:56.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.000778 17174 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:56.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 21 03:08:56 dist-test-slave-62xr krb5kdc[16362](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1774062536, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.247.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:56.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.009882 17174 init.cc:377] Logged in from keytab as kudu/127.15.247.189@KRBTEST.COM (short username kudu)
03:08:56.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.017481 17174 webserver.cc:492] Webserver started at http://127.15.247.189:36167/ using document root <none> and password file <none>
03:08:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.018982 17174 fs_manager.cc:362] Metadata directory not provided
03:08:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.019402 17174 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:56.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.030905 17174 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.005s sys 0.000s
03:08:56.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.038260 17232 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:56.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.040198 17174 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.004s sys 0.000s
03:08:56.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.040681 17174 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data,/tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/wal
03:08:56.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ce0e821f1a84309905e4b5361438a72"
03:08:56.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-21 03:08:33 on dist-test-slave-62xr"
03:08:56.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.043197 17174 fs_report.cc:389] FS layout report
03:08:56.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:56.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/wal
03:08:56.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/wal
03:08:56.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data/data
03:08:56.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:56.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:56.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:56.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:56.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:56.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:56.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:56.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:56.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:56.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:56.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:56.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.102391 17174 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:56.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.110157 17174 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:56.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.110708 17174 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:56.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.141640 17005 master_runner.cc:186] Error getting master registration for 127.15.247.189:35611: Network error: Client connection negotiation failed: client connection to 127.15.247.189:35611: connect: Connection refused (error 111),
03:08:56.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.142113 17005 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:56.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.245982 17174 rpc_server.cc:307] RPC server started. Bound to: 127.15.247.189:35611
03:08:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.245959 17284 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.247.189:35611 every 8 connection(s)
03:08:56.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.250689 17174 server_base.cc:1193] Dumped server information to /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/master-1/data/info.pb
03:08:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.256866 16350 external_mini_cluster.cc:1428] Started /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu as pid 17174
03:08:56.258 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:56.258 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.247.190:40563(127.15.247.190:40563)] connecting to peer
03:08:56.260 [DEBUG - kudu-nio-3] (Connection.java:757) [peer master-127.15.247.190:40563(127.15.247.190:40563)] Successfully connected to peer
03:08:56.269 [DEBUG - kudu-nio-3] (SecurityContext.java:222) Refreshing Kerberos credentials...
03:08:56.269 [DEBUG - kudu-nio-3] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task67Ldfr/test-tmp/mini-kudu-cluster15042157934236049238/krb5kdc/krb5cc
03:08:56.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.270048 17285 sys_catalog.cc:263] Verifying existing consensus state
03:08:56.271 [DEBUG - kudu-nio-3] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:08:56.271 [WARN - kudu-nio-3] (SecurityContext.java:231) Tried to refresh Kerberos credentials but was unable to re-login from ticket cache
03:08:56.271 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.15.247.190:40563(127.15.247.190:40563)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
03:08:56.273 [DEBUG - main] (Connection.java:575) [peer master-127.15.247.190:40563(127.15.247.190:40563)] disconnecting while in state TERMINATED
03:08:56.273 [DEBUG - main] (Connection.java:575) [peer master-127.15.247.189:35611(127.15.247.189:35611)] disconnecting while in state TERMINATED
03:08:56.273 [DEBUG - main] (Connection.java:575) [peer master-127.15.247.188:40735(127.15.247.188:40735)] disconnecting while in state TERMINATED
03:08:56.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:56.275235 17288 negotiation.cc:337] Failed RPC negotiation. Trace:
03:08:56.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0321 03:08:56.260943 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:50266 (local address 127.15.247.190:40563)
03:08:56.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0321 03:08:56.262670 (+ 1727us) server_negotiation.cc:207] Beginning negotiation
03:08:56.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0321 03:08:56.262704 (+ 34us) server_negotiation.cc:400] Waiting for connection header
03:08:56.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0321 03:08:56.262766 (+ 62us) server_negotiation.cc:408] Connection header received
03:08:56.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0321 03:08:56.267407 (+ 4641us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
03:08:56.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0321 03:08:56.267444 (+ 37us) server_negotiation.cc:462] Received NEGOTIATE request from client
03:08:56.276 [DEBUG - kudu-nio-3] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:08:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0321 03:08:56.267717 (+ 273us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
03:08:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0321 03:08:56.267930 (+ 213us) server_negotiation.cc:227] Negotiated authn=SASL
03:08:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0321 03:08:56.274138 (+ 6208us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:50266: BlockingRecv error: recv got EOF from 127.0.0.1:50266 (error 108)
03:08:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1034,"thread_start_us":761,"threads_started":1}
03:08:56.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.279788 17285 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2ce0e821f1a84309905e4b5361438a72: Bootstrap starting.
03:08:56.284 [DEBUG - kudu-nio-3] (Connection.java:239) [peer master-127.15.247.190:40563(127.15.247.190:40563)] handling channelInactive
03:08:56.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.286801 16350 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu with pid 17005
03:08:56.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.337718 16350 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu with pid 17174
03:08:56.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:56.358644 16350 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu with pid 17079
03:08:57.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-21T03:08:57Z chronyd exiting
03:08:58.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:58.299611 16335 debug-util.cc:398] Leaking SignalData structure 0x7b08000bac60 after lost signal to thread 16210
03:08:58.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260321 03:08:58.300590 16335 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6fc0 after lost signal to thread 16338
03:08:58.737 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:08:58.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:58.739962 15628 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu with pid 15928
03:08:58.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:58.767957 15628 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu with pid 16069
03:08:58.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:58.802347 15628 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu with pid 16209
03:08:58.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:58.840147 15628 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu with pid 15640
03:08:58.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:58.882202 15628 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu with pid 15710
03:08:58.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260321 03:08:58.935170 15628 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task67Ldfr/build/tsan/bin/kudu with pid 15785
03:09:00.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-21T03:09:00Z chronyd exiting
03:09:01.390 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
03:09:01.645 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: giving up after 1 attempts
03:09:01.645 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more