20:03:47.728 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:03:47.739 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskCk90tC/test-tmp
20:03:47.745 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:03:47.748 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskCk90tC/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:03:47.958 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:03:48.158 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:03:48.159 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:03:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:48.164208 27188 mini_oidc.cc:150] Starting JWKS server
20:03:48.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:48.168431 27188 webserver.cc:533] Webserver started at http://127.0.0.1:43411/ using document root <none> and password file <none>
20:03:48.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:48.169036 27188 mini_oidc.cc:186] Starting OIDC Discovery server
20:03:48.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:48.171173 27188 webserver.cc:533] Webserver started at http://127.0.0.1:43939/ using document root <none> and password file <none>
20:03:48.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-14T20:03:48Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:03:48.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-14T20:03:48Z Disabled control of system clock
20:03:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:48.238699 27188 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-0/wal
20:03:48.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-0/data
20:03:48.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:48.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:48.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:48.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:48.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:48.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:48.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-0/logs
20:03:48.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-0/data/info.pb
20:03:48.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:48.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:48.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:48.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:48.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:48.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:48.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:48.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:48.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:03:48.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:48.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:48.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:48.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.141.62:36061
20:03:48.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.141.62
20:03:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.141.20:33323
20:03:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:48.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:48.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:48.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:48.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.141.62:36061,127.26.141.61:44531,127.26.141.60:43105
20:03:48.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:48.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43411/jwks.json with env {}
20:03:48.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:48.572136 27200 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:48.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:48.573558 27200 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:48.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:48.574159 27200 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:48.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:48.609668 27200 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:03:48.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:48.610059 27200 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:48.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:48.610306 27200 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:03:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:48.610561 27200 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:03:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:48.610760 27200 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:03:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:48.610908 27200 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43411/jwks.json
20:03:48.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:48.651652 27200 master_runner.cc:387] Master server non-default flags:
20:03:48.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:48.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.141.20:33323
20:03:48.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:48.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:48.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-0/data
20:03:48.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-0/wal
20:03:48.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:48.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.141.62:36061,127.26.141.61:44531,127.26.141.60:43105
20:03:48.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:48.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:48.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:48.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.141.62:36061
20:03:48.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:48.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:48.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:48.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43411/jwks.json
20:03:48.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:48.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:48.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-0/data/info.pb
20:03:48.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.141.62
20:03:48.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:48.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:48.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:48.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:48.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:48.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:48.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-0/logs
20:03:48.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:48.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:48.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:03:48.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:03:48.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:48.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6b1bbe17b8ab938291541389a90c67c358bba716
20:03:48.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:48.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jun 2026 19:43:21 UTC on e7f111948823
20:03:48.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12395
20:03:48.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:48.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:48.653456 27200 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:48.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:48.656111 27200 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:48.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:48.672267 27206 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:50.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:48.672405 27207 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:50.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:50.076381 27205 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 27200
20:03:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:50.527618 27200 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.854s user 0.718s sys 1.135s
20:03:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:50.530599 27200 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.857s user 0.719s sys 1.135s
20:03:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:50.530815 27209 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:50.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:50.534343 27208 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1855 milliseconds
20:03:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.534384 27200 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:50.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.536022 27200 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.541771 27200 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:50.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.543237 27200 hybrid_clock.cc:648] HybridClock initialized: now 1781467430543193 us; error 70 us; skew 500 ppm
20:03:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.544293 27200 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:50.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.551458 27200 webserver.cc:533] Webserver started at http://127.26.141.62:33929/ using document root <none> and password file <none>
20:03:50.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.552461 27200 fs_manager.cc:362] Metadata directory not provided
20:03:50.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.552670 27200 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:50.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.553217 27200 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:50.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.558879 27200 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-0/data/instance:
20:03:50.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dec92a460e334d739f9441ea16cf789f"
20:03:50.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:03:50 on dist-test-slave-h0zs"
20:03:50.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.560379 27200 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-0/wal/instance:
20:03:50.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dec92a460e334d739f9441ea16cf789f"
20:03:50.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:03:50 on dist-test-slave-h0zs"
20:03:50.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.568782 27200 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.010s sys 0.000s
20:03:50.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.575464 27216 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:03:50.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.576984 27200 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
20:03:50.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.577368 27200 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-0/data,/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-0/wal
20:03:50.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dec92a460e334d739f9441ea16cf789f"
20:03:50.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:03:50 on dist-test-slave-h0zs"
20:03:50.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.577811 27200 fs_report.cc:389] FS layout report
20:03:50.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:50.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-0/wal
20:03:50.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-0/wal
20:03:50.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-0/data/data
20:03:50.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:50.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:50.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:50.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:50.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:50.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:50.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:50.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:50.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:50.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:50.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:50.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.633772 27200 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:03:50.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.635537 27200 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:50.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.635974 27200 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:50.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.720863 27200 rpc_server.cc:307] RPC server started. Bound to: 127.26.141.62:36061
20:03:50.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.720942 27267 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.141.62:36061 every 8 connection(s)
20:03:50.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.724109 27200 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-0/data/info.pb
20:03:50.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.731607 27268 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:50.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.732861 27188 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu as pid 27200
20:03:50.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.733453 27188 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-0/wal/instance
20:03:50.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.739172 27188 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:03:50.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:03:50.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-1/wal
20:03:50.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-1/data
20:03:50.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:50.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:50.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:50.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:50.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:50.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:50.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-1/logs
20:03:50.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-1/data/info.pb
20:03:50.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:50.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:50.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:50.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:50.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:50.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:50.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:50.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:50.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:03:50.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:50.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:50.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:50.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.141.61:44531
20:03:50.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.141.61
20:03:50.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:50.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.141.20:33323
20:03:50.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:50.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:50.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:50.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:50.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.141.62:36061,127.26.141.61:44531,127.26.141.60:43105
20:03:50.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:50.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43411/jwks.json with env {}
20:03:50.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.748914 27268 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } has no permanent_uuid. Determining permanent_uuid...
20:03:50.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.787398 27268 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } has no permanent_uuid. Determining permanent_uuid...
20:03:50.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:50.790755 27219 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.141.61:44531: connect: Connection refused (error 111)
20:03:50.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:50.795377 27268 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.61:44531: Network error: Client connection negotiation failed: client connection to 127.26.141.61:44531: connect: Connection refused (error 111)
20:03:50.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.844982 27268 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } attempt: 1
20:03:50.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:50.850309 27268 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.61:44531: Network error: Client connection negotiation failed: client connection to 127.26.141.61:44531: connect: Connection refused (error 111)
20:03:50.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:50.918754 27268 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } attempt: 2
20:03:50.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:50.924117 27268 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.61:44531: Network error: Client connection negotiation failed: client connection to 127.26.141.61:44531: connect: Connection refused (error 111)
20:03:51.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:51.015585 27268 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } attempt: 3
20:03:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:51.020361 27268 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.61:44531: Network error: Client connection negotiation failed: client connection to 127.26.141.61:44531: connect: Connection refused (error 111)
20:03:51.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:51.097730 27270 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:51.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:51.098412 27270 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:51.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:51.098874 27270 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:51.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:51.134267 27270 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:03:51.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:51.134569 27270 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:51.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:51.134764 27270 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:03:51.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:51.134956 27270 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:03:51.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:51.135154 27270 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:03:51.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:51.135304 27270 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43411/jwks.json
20:03:51.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:51.163861 27268 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } attempt: 4
20:03:51.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:51.169094 27268 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.61:44531: Network error: Client connection negotiation failed: client connection to 127.26.141.61:44531: connect: Connection refused (error 111)
20:03:51.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:51.179579 27270 master_runner.cc:387] Master server non-default flags:
20:03:51.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:51.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.141.20:33323
20:03:51.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:51.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:51.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-1/data
20:03:51.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-1/wal
20:03:51.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:51.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.141.62:36061,127.26.141.61:44531,127.26.141.60:43105
20:03:51.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:51.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:51.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:51.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.141.61:44531
20:03:51.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:51.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:51.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:51.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43411/jwks.json
20:03:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-1/data/info.pb
20:03:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.141.61
20:03:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-1/logs
20:03:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:03:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:03:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:51.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6b1bbe17b8ab938291541389a90c67c358bba716
20:03:51.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:51.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jun 2026 19:43:21 UTC on e7f111948823
20:03:51.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12395
20:03:51.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:51.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:51.181298 27270 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:51.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:51.184289 27270 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:51.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:51.197747 27279 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:51.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:51.468607 27268 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } attempt: 5
20:03:51.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:51.476620 27268 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.61:44531: Network error: Client connection negotiation failed: client connection to 127.26.141.61:44531: connect: Connection refused (error 111)
20:03:52.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.024147 27268 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } attempt: 6
20:03:52.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:52.049458 27268 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.61:44531: Network error: Client connection negotiation failed: client connection to 127.26.141.61:44531: connect: Connection refused (error 111)
20:03:52.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:52.602185 27278 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 27270
20:03:52.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:51.198243 27280 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:52.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:52.680863 27270 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.481s user 0.507s sys 0.966s
20:03:52.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:52.681957 27270 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.484s user 0.507s sys 0.966s
20:03:52.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:52.682816 27282 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:52.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:52.686005 27281 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1487 milliseconds
20:03:52.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.686050 27270 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:52.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.687405 27270 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:52.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.690166 27270 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.691550 27270 hybrid_clock.cc:648] HybridClock initialized: now 1781467432691482 us; error 86 us; skew 500 ppm
20:03:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.692879 27270 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:52.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.700217 27270 webserver.cc:533] Webserver started at http://127.26.141.61:34429/ using document root <none> and password file <none>
20:03:52.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.701542 27270 fs_manager.cc:362] Metadata directory not provided
20:03:52.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.701761 27270 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:52.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.702198 27270 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:52.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.707576 27270 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-1/data/instance:
20:03:52.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b94b167ed049455c871a5a2974aec730"
20:03:52.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:03:52 on dist-test-slave-h0zs"
20:03:52.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.708810 27270 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-1/wal/instance:
20:03:52.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b94b167ed049455c871a5a2974aec730"
20:03:52.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:03:52 on dist-test-slave-h0zs"
20:03:52.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.717337 27270 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
20:03:52.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.726261 27291 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:03:52.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.727876 27270 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.002s sys 0.002s
20:03:52.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.728323 27270 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-1/data,/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-1/wal
20:03:52.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b94b167ed049455c871a5a2974aec730"
20:03:52.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:03:52 on dist-test-slave-h0zs"
20:03:52.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.728817 27270 fs_report.cc:389] FS layout report
20:03:52.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:52.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-1/wal
20:03:52.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-1/wal
20:03:52.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-1/data/data
20:03:52.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:52.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:52.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:52.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:52.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:52.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:52.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.784719 27270 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:03:52.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.786593 27270 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:52.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.787055 27270 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:52.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.871505 27270 rpc_server.cc:307] RPC server started. Bound to: 127.26.141.61:44531
20:03:52.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.871591 27342 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.141.61:44531 every 8 connection(s)
20:03:52.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.874845 27270 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-1/data/info.pb
20:03:52.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.878597 27188 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu as pid 27270
20:03:52.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.879185 27188 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-1/wal/instance
20:03:52.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.882903 27188 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:03:52.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:03:52.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-2/wal
20:03:52.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-2/data
20:03:52.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:52.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-2/logs
20:03:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-2/data/info.pb
20:03:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:52.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:52.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:52.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:03:52.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:52.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:52.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:52.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.141.60:43105
20:03:52.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.141.60
20:03:52.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:52.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.141.20:33323
20:03:52.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:52.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:52.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:52.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:52.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.141.62:36061,127.26.141.61:44531,127.26.141.60:43105
20:03:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43411/jwks.json with env {}
20:03:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.884717 27343 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:52.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.906800 27343 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } has no permanent_uuid. Determining permanent_uuid...
20:03:52.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.937201 27343 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } has no permanent_uuid. Determining permanent_uuid...
20:03:52.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:52.952862 27343 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } has no permanent_uuid. Determining permanent_uuid...
20:03:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:52.956012 27293 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.141.60:43105: connect: Connection refused (error 111)
20:03:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:52.960660 27343 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.60:43105: Network error: Client connection negotiation failed: client connection to 127.26.141.60:43105: connect: Connection refused (error 111)
20:03:53.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:53.010247 27343 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } attempt: 1
20:03:53.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.015659 27343 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.60:43105: Network error: Client connection negotiation failed: client connection to 127.26.141.60:43105: connect: Connection refused (error 111)
20:03:53.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:53.084196 27343 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } attempt: 2
20:03:53.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.089263 27343 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.60:43105: Network error: Client connection negotiation failed: client connection to 127.26.141.60:43105: connect: Connection refused (error 111)
20:03:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:53.110190 27268 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } attempt: 7
20:03:53.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:53.124089 27268 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } has no permanent_uuid. Determining permanent_uuid...
20:03:53.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.130955 27268 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.60:43105: Network error: Client connection negotiation failed: client connection to 127.26.141.60:43105: connect: Connection refused (error 111)
20:03:53.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:53.180788 27343 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } attempt: 3
20:03:53.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.185904 27343 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.60:43105: Network error: Client connection negotiation failed: client connection to 127.26.141.60:43105: connect: Connection refused (error 111)
20:03:53.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:53.189491 27268 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } attempt: 1
20:03:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.194156 27268 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.60:43105: Network error: Client connection negotiation failed: client connection to 127.26.141.60:43105: connect: Connection refused (error 111)
20:03:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.258301 27345 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.258942 27345 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.259541 27345 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:53.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:53.275648 27268 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } attempt: 2
20:03:53.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.280725 27268 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.60:43105: Network error: Client connection negotiation failed: client connection to 127.26.141.60:43105: connect: Connection refused (error 111)
20:03:53.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.304388 27345 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:03:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.304771 27345 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.305017 27345 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:03:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.305341 27345 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:03:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.305615 27345 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:03:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.305827 27345 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43411/jwks.json
20:03:53.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:53.329419 27343 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } attempt: 4
20:03:53.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.334476 27343 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.60:43105: Network error: Client connection negotiation failed: client connection to 127.26.141.60:43105: connect: Connection refused (error 111)
20:03:53.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:53.349119 27345 master_runner.cc:387] Master server non-default flags:
20:03:53.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:53.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.141.20:33323
20:03:53.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:53.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:53.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-2/data
20:03:53.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-2/wal
20:03:53.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:53.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.141.62:36061,127.26.141.61:44531,127.26.141.60:43105
20:03:53.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:53.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:53.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:53.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.141.60:43105
20:03:53.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:53.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:53.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:53.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43411/jwks.json
20:03:53.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:53.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:53.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-2/data/info.pb
20:03:53.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.141.60
20:03:53.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:53.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:53.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:53.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:53.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:53.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:53.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-2/logs
20:03:53.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:03:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:03:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6b1bbe17b8ab938291541389a90c67c358bba716
20:03:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jun 2026 19:43:21 UTC on e7f111948823
20:03:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12395
20:03:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:53.351060 27345 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:53.353102 27345 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:53.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.365016 27357 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:53.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:53.366199 27268 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } attempt: 3
20:03:53.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.373200 27268 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.60:43105: Network error: Client connection negotiation failed: client connection to 127.26.141.60:43105: connect: Connection refused (error 111)
20:03:53.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:53.513659 27268 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } attempt: 4
20:03:53.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.518977 27268 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.60:43105: Network error: Client connection negotiation failed: client connection to 127.26.141.60:43105: connect: Connection refused (error 111)
20:03:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:53.633982 27343 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } attempt: 5
20:03:53.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.639415 27343 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.60:43105: Network error: Client connection negotiation failed: client connection to 127.26.141.60:43105: connect: Connection refused (error 111)
20:03:53.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:53.802708 27268 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } attempt: 5
20:03:53.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.808697 27268 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.60:43105: Network error: Client connection negotiation failed: client connection to 127.26.141.60:43105: connect: Connection refused (error 111)
20:03:54.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.187119 27343 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } attempt: 6
20:03:54.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:54.195988 27343 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.60:43105: Network error: Client connection negotiation failed: client connection to 127.26.141.60:43105: connect: Connection refused (error 111)
20:03:54.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.361397 27268 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } attempt: 6
20:03:54.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:54.371430 27268 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.141.60:43105: Network error: Client connection negotiation failed: client connection to 127.26.141.60:43105: connect: Connection refused (error 111)
20:03:54.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.370849 27358 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:54.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:53.371474 27360 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:54.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:53.372603 27345 server_base.cc:1061] running on GCE node
20:03:54.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.632997 27345 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.636582 27345 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:54.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.638123 27345 hybrid_clock.cc:648] HybridClock initialized: now 1781467434638091 us; error 52 us; skew 500 ppm
20:03:54.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.639005 27345 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:54.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.646380 27345 webserver.cc:533] Webserver started at http://127.26.141.60:40785/ using document root <none> and password file <none>
20:03:54.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.647394 27345 fs_manager.cc:362] Metadata directory not provided
20:03:54.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.647590 27345 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:54.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.648022 27345 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:54.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.652913 27345 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-2/data/instance:
20:03:54.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b41d7f7ede9409cbbb680cb421341df"
20:03:54.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:03:54 on dist-test-slave-h0zs"
20:03:54.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.654201 27345 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-2/wal/instance:
20:03:54.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b41d7f7ede9409cbbb680cb421341df"
20:03:54.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:03:54 on dist-test-slave-h0zs"
20:03:54.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.662060 27345 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
20:03:54.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.668545 27371 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:03:54.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.670135 27345 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
20:03:54.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.670549 27345 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-2/data,/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-2/wal
20:03:54.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b41d7f7ede9409cbbb680cb421341df"
20:03:54.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:03:54 on dist-test-slave-h0zs"
20:03:54.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.670949 27345 fs_report.cc:389] FS layout report
20:03:54.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:54.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-2/wal
20:03:54.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-2/wal
20:03:54.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-2/data/data
20:03:54.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:54.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:54.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:54.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:54.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:54.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:54.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:54.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:54.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:54.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:54.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:54.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.729225 27345 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:03:54.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.731125 27345 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:54.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.731721 27345 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:54.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.816576 27345 rpc_server.cc:307] RPC server started. Bound to: 127.26.141.60:43105
20:03:54.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.816649 27422 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.141.60:43105 every 8 connection(s)
20:03:54.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.820039 27345 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-2/data/info.pb
20:03:54.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.823758 27188 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu as pid 27345
20:03:54.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.824343 27188 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/master-2/wal/instance
20:03:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.828469 27423 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.846608 27423 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } has no permanent_uuid. Determining permanent_uuid...
20:03:54.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.876896 27423 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } has no permanent_uuid. Determining permanent_uuid...
20:03:54.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.889436 27423 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } has no permanent_uuid. Determining permanent_uuid...
20:03:54.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.909369 27423 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df: Bootstrap starting.
20:03:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.917513 27423 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df: Neither blocks nor log segments found. Creating new log.
20:03:54.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.919766 27423 log.cc:826] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df: Log is configured to *not* fsync() on all Append() calls
20:03:54.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.925925 27423 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df: No bootstrap required, opened a new log
20:03:54.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.947925 27423 raft_consensus.cc:359] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } }
20:03:54.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.948772 27423 raft_consensus.cc:385] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:54.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.949060 27423 raft_consensus.cc:740] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0b41d7f7ede9409cbbb680cb421341df, State: Initialized, Role: FOLLOWER
20:03:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.950341 27423 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } }
20:03:54.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.954408 27432 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } } }
20:03:54.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.955271 27432 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [sys.catalog]: This master's current role is: FOLLOWER
20:03:54.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.956321 27423 sys_catalog.cc:565] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [sys.catalog]: configured and running, proceeding with master startup.
20:03:54.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:54.986256 27443 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:03:54.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:54.987694 27377 tablet.cc:2404] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:54.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:54.989691 27443 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:03:55.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:54.982172 27345 master_runner.cc:187] Error getting master registration for 127.26.141.62:36061: OK, instance_id { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" instance_seqno: 1781467430694939 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:03:55.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.009440 27345 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:03:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.256669 27343 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } attempt: 7
20:03:55.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.279277 27343 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730: Bootstrap starting.
20:03:55.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.287242 27343 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730: Neither blocks nor log segments found. Creating new log.
20:03:55.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.289721 27343 log.cc:826] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730: Log is configured to *not* fsync() on all Append() calls
20:03:55.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.295006 27343 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730: No bootstrap required, opened a new log
20:03:55.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.317396 27343 raft_consensus.cc:359] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } }
20:03:55.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.318135 27343 raft_consensus.cc:385] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:55.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.318410 27343 raft_consensus.cc:740] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b94b167ed049455c871a5a2974aec730, State: Initialized, Role: FOLLOWER
20:03:55.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.319449 27343 consensus_queue.cc:260] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } }
20:03:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.322111 27447 sys_catalog.cc:455] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } } }
20:03:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.322944 27447 sys_catalog.cc:458] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730 [sys.catalog]: This master's current role is: FOLLOWER
20:03:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.323639 27343 sys_catalog.cc:565] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730 [sys.catalog]: configured and running, proceeding with master startup.
20:03:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:55.359146 27458 catalog_manager.cc:1593] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:03:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:55.360252 27458 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:03:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.352449 27270 master_runner.cc:187] Error getting master registration for 127.26.141.62:36061: OK, instance_id { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" instance_seqno: 1781467430694939 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:03:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.385751 27270 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:03:55.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.405134 27268 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } attempt: 7
20:03:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.408823 27432 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:03:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.409523 27432 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } }
20:03:55.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.417011 27318 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0b41d7f7ede9409cbbb680cb421341df" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b94b167ed049455c871a5a2974aec730" is_pre_election: true
20:03:55.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.420851 27318 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0b41d7f7ede9409cbbb680cb421341df in term 0.
20:03:55.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.424216 27374 leader_election.cc:304] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0b41d7f7ede9409cbbb680cb421341df, b94b167ed049455c871a5a2974aec730; no voters:
20:03:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.430892 27461 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [term 0 FOLLOWER]: Leader pre-election won for term 1
20:03:55.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.431852 27461 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:03:55.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.432566 27461 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [term 0 FOLLOWER]: Advancing to term 1
20:03:55.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.440852 27461 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } }
20:03:55.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.441326 27432 leader_election.cc:290] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers dec92a460e334d739f9441ea16cf789f (127.26.141.62:36061), b94b167ed049455c871a5a2974aec730 (127.26.141.61:44531)
20:03:55.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.444986 27461 leader_election.cc:290] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [CANDIDATE]: Term 1 election: Requested vote from peers dec92a460e334d739f9441ea16cf789f (127.26.141.62:36061), b94b167ed049455c871a5a2974aec730 (127.26.141.61:44531)
20:03:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.446934 27318 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0b41d7f7ede9409cbbb680cb421341df" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b94b167ed049455c871a5a2974aec730"
20:03:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.447685 27318 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730 [term 0 FOLLOWER]: Advancing to term 1
20:03:55.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.460388 27268 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f: Bootstrap starting.
20:03:55.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.467566 27318 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0b41d7f7ede9409cbbb680cb421341df in term 1.
20:03:55.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.476933 27374 leader_election.cc:304] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0b41d7f7ede9409cbbb680cb421341df, b94b167ed049455c871a5a2974aec730; no voters:
20:03:55.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.479212 27461 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [term 1 FOLLOWER]: Leader election won for term 1
20:03:55.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.481298 27268 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f: Neither blocks nor log segments found. Creating new log.
20:03:55.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.480957 27461 raft_consensus.cc:697] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [term 1 LEADER]: Becoming Leader. State: Replica: 0b41d7f7ede9409cbbb680cb421341df, State: Running, Role: LEADER
20:03:55.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.484511 27268 log.cc:826] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f: Log is configured to *not* fsync() on all Append() calls
20:03:55.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.483325 27461 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } }
20:03:55.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.512066 27432 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0b41d7f7ede9409cbbb680cb421341df. Latest consensus state: current_term: 1 leader_uuid: "0b41d7f7ede9409cbbb680cb421341df" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } } }
20:03:55.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.515920 27432 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [sys.catalog]: This master's current role is: LEADER
20:03:55.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.517743 27268 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f: No bootstrap required, opened a new log
20:03:55.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.531577 27464 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
20:03:55.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:55.546386 27297 tablet.cc:2404] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:55.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.549667 27464 catalog_manager.cc:1519] Initializing Kudu cluster ID...
20:03:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.574186 27268 raft_consensus.cc:359] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } }
20:03:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.575956 27268 raft_consensus.cc:385] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:55.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.576723 27268 raft_consensus.cc:740] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dec92a460e334d739f9441ea16cf789f, State: Initialized, Role: FOLLOWER
20:03:55.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.581189 27318 raft_consensus.cc:1275] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730 [term 1 FOLLOWER]: Refusing update from remote peer 0b41d7f7ede9409cbbb680cb421341df: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:03:55.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.580871 27268 consensus_queue.cc:260] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } }
20:03:55.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.584220 27432 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [LEADER]: Connected to new peer: Peer: permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:03:55.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.592262 27268 sys_catalog.cc:565] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f [sys.catalog]: configured and running, proceeding with master startup.
20:03:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.594559 27243 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0b41d7f7ede9409cbbb680cb421341df" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dec92a460e334d739f9441ea16cf789f" is_pre_election: true
20:03:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.595763 27242 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0b41d7f7ede9409cbbb680cb421341df" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dec92a460e334d739f9441ea16cf789f"
20:03:55.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.597430 27466 sys_catalog.cc:455] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } } }
20:03:55.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.598629 27466 sys_catalog.cc:458] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f [sys.catalog]: This master's current role is: FOLLOWER
20:03:55.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.598542 27243 raft_consensus.cc:2468] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0b41d7f7ede9409cbbb680cb421341df in term 0.
20:03:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.612869 27241 raft_consensus.cc:3060] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f [term 0 FOLLOWER]: Advancing to term 1
20:03:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.621924 27241 raft_consensus.cc:1275] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f [term 1 FOLLOWER]: Refusing update from remote peer 0b41d7f7ede9409cbbb680cb421341df: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:03:55.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.630302 27432 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [LEADER]: Connected to new peer: Peer: permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:03:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:55.648000 27222 tablet.cc:2404] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:55.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.653986 27447 sys_catalog.cc:455] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0b41d7f7ede9409cbbb680cb421341df. Latest consensus state: current_term: 1 leader_uuid: "0b41d7f7ede9409cbbb680cb421341df" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } } }
20:03:55.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.654785 27447 sys_catalog.cc:458] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730 [sys.catalog]: This master's current role is: FOLLOWER
20:03:55.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.682210 27447 sys_catalog.cc:455] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0b41d7f7ede9409cbbb680cb421341df" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } } }
20:03:55.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.683187 27447 sys_catalog.cc:458] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730 [sys.catalog]: This master's current role is: FOLLOWER
20:03:55.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.711159 27461 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0b41d7f7ede9409cbbb680cb421341df" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } } }
20:03:55.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.712311 27461 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [sys.catalog]: This master's current role is: LEADER
20:03:55.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.737708 27464 catalog_manager.cc:1382] Generated new cluster ID: 7d8f742cef054cf1bd12f9b95ba5b4c4
20:03:55.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.738063 27464 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
20:03:55.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.739804 27466 sys_catalog.cc:455] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0b41d7f7ede9409cbbb680cb421341df. Latest consensus state: current_term: 1 leader_uuid: "0b41d7f7ede9409cbbb680cb421341df" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } } }
20:03:55.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.741918 27466 sys_catalog.cc:458] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f [sys.catalog]: This master's current role is: FOLLOWER
20:03:55.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.746912 27432 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0b41d7f7ede9409cbbb680cb421341df" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } } }
20:03:55.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.747898 27432 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df [sys.catalog]: This master's current role is: LEADER
20:03:55.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.754369 27473 mvcc.cc:204] Tried to move back new op lower bound from 7296890616106602496 to 7296890615766437888. Current Snapshot: MvccSnapshot[applied={T|T < 7296890616106602496}]
20:03:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.759270 27466 sys_catalog.cc:455] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0b41d7f7ede9409cbbb680cb421341df" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dec92a460e334d739f9441ea16cf789f" member_type: VOTER last_known_addr { host: "127.26.141.62" port: 36061 } } peers { permanent_uuid: "b94b167ed049455c871a5a2974aec730" member_type: VOTER last_known_addr { host: "127.26.141.61" port: 44531 } } peers { permanent_uuid: "0b41d7f7ede9409cbbb680cb421341df" member_type: VOTER last_known_addr { host: "127.26.141.60" port: 43105 } } }
20:03:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.760027 27466 sys_catalog.cc:458] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f [sys.catalog]: This master's current role is: FOLLOWER
20:03:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.810066 27485 catalog_manager.cc:1294] Loaded cluster ID: 7d8f742cef054cf1bd12f9b95ba5b4c4
20:03:55.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.821053 27464 catalog_manager.cc:1405] Generated new certificate authority record
20:03:55.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.823452 27464 catalog_manager.cc:1539] Loading token signing keys...
20:03:55.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.825284 27485 catalog_manager.cc:1587] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f: loading cluster ID for follower catalog manager: success
20:03:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.878553 27464 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 0b41d7f7ede9409cbbb680cb421341df: Generated new TSK 0
20:03:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.881788 27464 catalog_manager.cc:1549] Initializing in-progress tserver states...
20:03:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.887724 27485 catalog_manager.cc:1609] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f: acquiring CA information for follower catalog manager: success
20:03:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.915060 27485 catalog_manager.cc:1637] T 00000000000000000000000000000000 P dec92a460e334d739f9441ea16cf789f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:03:55.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:55.933205 27188 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:03:55.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:03:55.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-0/wal
20:03:55.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-0/data
20:03:55.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:55.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:55.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:55.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:55.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:55.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:55.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-0/logs
20:03:55.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-0/data/info.pb
20:03:55.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:55.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:55.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:55.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:55.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:55.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:55.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:55.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:55.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:55.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:55.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.141.1:0
20:03:55.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.141.1
20:03:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.141.1
20:03:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.141.62:36061,127.26.141.61:44531,127.26.141.60:43105
20:03:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.141.20:33323
20:03:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:56.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:56.367406 27458 catalog_manager.cc:1294] Loaded cluster ID: 7d8f742cef054cf1bd12f9b95ba5b4c4
20:03:56.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:56.368698 27458 catalog_manager.cc:1587] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730: loading cluster ID for follower catalog manager: success
20:03:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:56.395623 27458 catalog_manager.cc:1609] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730: acquiring CA information for follower catalog manager: success
20:03:56.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:56.406955 27458 catalog_manager.cc:1637] T 00000000000000000000000000000000 P b94b167ed049455c871a5a2974aec730: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:03:56.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:56.588828 27487 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:56.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:56.590390 27487 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:56.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:56.591595 27487 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:56.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:56.659991 27487 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:56.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:56.662119 27487 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.141.1
20:03:56.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:56.739841 27487 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:56.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:56.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.141.20:33323
20:03:56.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:56.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:56.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-0/data
20:03:56.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-0/wal
20:03:56.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:56.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:56.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.141.1:0
20:03:56.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:56.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:56.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:56.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-0/data/info.pb
20:03:56.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.141.1
20:03:56.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:56.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.141.62:36061,127.26.141.61:44531,127.26.141.60:43105
20:03:56.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:56.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:56.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:56.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:56.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:56.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.141.1
20:03:56.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-0/logs
20:03:56.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:56.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:56.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:03:56.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:56.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:56.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6b1bbe17b8ab938291541389a90c67c358bba716
20:03:56.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:56.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jun 2026 19:43:21 UTC on e7f111948823
20:03:56.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12395
20:03:56.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:56.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:56.745429 27487 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:56.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:56.748435 27487 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:56.806881 27493 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:58.187004 27492 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 27487
20:03:59.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:59.339838 27487 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.514s user 0.001s sys 0.000s
20:03:59.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:59.340229 27487 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.515s user 0.001s sys 0.000s
20:03:59.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:56.823777 27494 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:59.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:03:59.345258 27500 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:59.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.350564 27487 server_base.cc:1061] running on GCE node
20:03:59.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.352120 27487 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.354933 27487 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.356388 27487 hybrid_clock.cc:648] HybridClock initialized: now 1781467439356347 us; error 40 us; skew 500 ppm
20:03:59.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.357683 27487 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:59.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.366137 27487 webserver.cc:533] Webserver started at http://127.26.141.1:41399/ using document root <none> and password file <none>
20:03:59.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.367489 27487 fs_manager.cc:362] Metadata directory not provided
20:03:59.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.367796 27487 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:59.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.368585 27487 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.375770 27487 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-0/data/instance:
20:03:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "54553eeb8a294ee48c3f0e5643743604"
20:03:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:03:59 on dist-test-slave-h0zs"
20:03:59.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.377449 27487 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-0/wal/instance:
20:03:59.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "54553eeb8a294ee48c3f0e5643743604"
20:03:59.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:03:59 on dist-test-slave-h0zs"
20:03:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.388170 27487 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.009s sys 0.000s
20:03:59.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.396438 27505 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:03:59.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.398150 27487 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.007s sys 0.000s
20:03:59.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.398607 27487 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-0/data,/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-0/wal
20:03:59.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "54553eeb8a294ee48c3f0e5643743604"
20:03:59.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:03:59 on dist-test-slave-h0zs"
20:03:59.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.399121 27487 fs_report.cc:389] FS layout report
20:03:59.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:59.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-0/wal
20:03:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-0/wal
20:03:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-0/data/data
20:03:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.469460 27487 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:03:59.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.471690 27487 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:59.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.472470 27487 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.476924 27487 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:59.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.482540 27487 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:03:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.482810 27487 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:03:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.483142 27487 ts_tablet_manager.cc:616] Registered 0 tablets
20:03:59.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.483385 27487 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:03:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.679811 27487 rpc_server.cc:307] RPC server started. Bound to: 127.26.141.1:44173
20:03:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.679911 27617 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.141.1:44173 every 8 connection(s)
20:03:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.682807 27487 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-0/data/info.pb
20:03:59.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.688017 27188 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu as pid 27487
20:03:59.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.688943 27188 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-0/wal/instance
20:03:59.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.745839 27188 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:03:59.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:03:59.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-1/wal
20:03:59.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-1/data
20:03:59.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:59.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:59.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:59.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:59.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:59.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:59.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-1/logs
20:03:59.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-1/data/info.pb
20:03:59.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:59.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:59.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:59.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:59.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:59.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:59.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:59.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:59.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:59.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:59.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.141.2:0
20:03:59.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.141.2
20:03:59.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.141.2
20:03:59.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:59.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.141.62:36061,127.26.141.61:44531,127.26.141.60:43105
20:03:59.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.141.20:33323
20:03:59.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:59.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:59.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.761921 27618 heartbeater.cc:344] Connected to a master server at 127.26.141.60:43105
20:03:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.762447 27618 heartbeater.cc:461] Registering TS with master...
20:03:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.763916 27618 heartbeater.cc:507] Master 127.26.141.60:43105 requested a full tablet report, sending...
20:03:59.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.767688 27388 ts_manager.cc:194] Registered new tserver with Master: 54553eeb8a294ee48c3f0e5643743604 (127.26.141.1:44173)
20:03:59.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.771654 27388 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.141.1:37399
20:03:59.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.782482 27619 heartbeater.cc:344] Connected to a master server at 127.26.141.61:44531
20:03:59.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.782899 27619 heartbeater.cc:461] Registering TS with master...
20:03:59.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.783645 27619 heartbeater.cc:507] Master 127.26.141.61:44531 requested a full tablet report, sending...
20:03:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.787513 27308 ts_manager.cc:194] Registered new tserver with Master: 54553eeb8a294ee48c3f0e5643743604 (127.26.141.1:44173)
20:03:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.792903 27620 heartbeater.cc:344] Connected to a master server at 127.26.141.62:36061
20:03:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.793351 27620 heartbeater.cc:461] Registering TS with master...
20:03:59.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.794072 27620 heartbeater.cc:507] Master 127.26.141.62:36061 requested a full tablet report, sending...
20:03:59.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:03:59.797729 27233 ts_manager.cc:194] Registered new tserver with Master: 54553eeb8a294ee48c3f0e5643743604 (127.26.141.1:44173)
20:04:00.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:00.179026 27628 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:00.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:00.179724 27628 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:00.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:00.180431 27628 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:00.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:00.217293 27628 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:00.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:00.218410 27628 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.141.2
20:04:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:00.258675 27628 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.141.20:33323
20:04:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-1/data
20:04:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-1/wal
20:04:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.141.2:0
20:04:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-1/data/info.pb
20:04:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.141.2
20:04:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.141.62:36061,127.26.141.61:44531,127.26.141.60:43105
20:04:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.141.2
20:04:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-1/logs
20:04:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6b1bbe17b8ab938291541389a90c67c358bba716
20:04:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jun 2026 19:43:21 UTC on e7f111948823
20:04:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12395
20:04:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:00.260643 27628 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:00.263401 27628 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:00.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:00.281672 27634 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:00.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:00.777537 27618 heartbeater.cc:499] Master 127.26.141.60:43105 was elected leader, sending a full tablet report...
20:04:01.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:00.283564 27635 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:01.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:01.559907 27636 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1275 milliseconds
20:04:01.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:01.560250 27637 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:01.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.560248 27628 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:01.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.564566 27628 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.572456 27628 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.573940 27628 hybrid_clock.cc:648] HybridClock initialized: now 1781467441573862 us; error 86 us; skew 500 ppm
20:04:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.574919 27628 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.581933 27628 webserver.cc:533] Webserver started at http://127.26.141.2:43779/ using document root <none> and password file <none>
20:04:01.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.583024 27628 fs_manager.cc:362] Metadata directory not provided
20:04:01.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.583276 27628 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:01.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.583871 27628 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.588749 27628 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-1/data/instance:
20:04:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e608fab6a3748199fb5aff7ba63f6ff"
20:04:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:01 on dist-test-slave-h0zs"
20:04:01.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.590066 27628 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-1/wal/instance:
20:04:01.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e608fab6a3748199fb5aff7ba63f6ff"
20:04:01.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:01 on dist-test-slave-h0zs"
20:04:01.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.598079 27628 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
20:04:01.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.604295 27647 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:01.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.605476 27628 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:04:01.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.605834 27628 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-1/data,/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-1/wal
20:04:01.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e608fab6a3748199fb5aff7ba63f6ff"
20:04:01.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:01 on dist-test-slave-h0zs"
20:04:01.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.606263 27628 fs_report.cc:389] FS layout report
20:04:01.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-1/wal
20:04:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-1/wal
20:04:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-1/data/data
20:04:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.660503 27628 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:01.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.662349 27628 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:01.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.662918 27628 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:01.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.665910 27628 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:01.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.670878 27628 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:04:01.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.671128 27628 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:04:01.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.671484 27628 ts_tablet_manager.cc:616] Registered 0 tablets
20:04:01.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.671672 27628 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:04:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.867692 27628 rpc_server.cc:307] RPC server started. Bound to: 127.26.141.2:42225
20:04:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.867865 27759 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.141.2:42225 every 8 connection(s)
20:04:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.871424 27628 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-1/data/info.pb
20:04:01.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.882477 27188 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu as pid 27628
20:04:01.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.883078 27188 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-1/wal/instance
20:04:01.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.904341 27188 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-2/wal
20:04:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-2/data
20:04:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-2/logs
20:04:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-2/data/info.pb
20:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:01.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.141.3:0
20:04:01.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.141.3
20:04:01.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.141.3
20:04:01.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:01.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.141.62:36061,127.26.141.61:44531,127.26.141.60:43105
20:04:01.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.141.20:33323
20:04:01.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:01.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:04:01.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.946458 27762 heartbeater.cc:344] Connected to a master server at 127.26.141.62:36061
20:04:01.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.947019 27762 heartbeater.cc:461] Registering TS with master...
20:04:01.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.948508 27762 heartbeater.cc:507] Master 127.26.141.62:36061 requested a full tablet report, sending...
20:04:01.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.948850 27761 heartbeater.cc:344] Connected to a master server at 127.26.141.61:44531
20:04:01.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.949234 27761 heartbeater.cc:461] Registering TS with master...
20:04:01.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.950198 27761 heartbeater.cc:507] Master 127.26.141.61:44531 requested a full tablet report, sending...
20:04:01.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.950408 27760 heartbeater.cc:344] Connected to a master server at 127.26.141.60:43105
20:04:01.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.950723 27760 heartbeater.cc:461] Registering TS with master...
20:04:01.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.951629 27760 heartbeater.cc:507] Master 127.26.141.60:43105 requested a full tablet report, sending...
20:04:01.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.952425 27233 ts_manager.cc:194] Registered new tserver with Master: 2e608fab6a3748199fb5aff7ba63f6ff (127.26.141.2:42225)
20:04:01.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.954972 27388 ts_manager.cc:194] Registered new tserver with Master: 2e608fab6a3748199fb5aff7ba63f6ff (127.26.141.2:42225)
20:04:01.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.955075 27308 ts_manager.cc:194] Registered new tserver with Master: 2e608fab6a3748199fb5aff7ba63f6ff (127.26.141.2:42225)
20:04:01.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:01.956714 27388 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.141.2:34541
20:04:02.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:02.312286 27770 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:02.312923 27770 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:02.313429 27770 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:02.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:02.350948 27770 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:02.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:02.351801 27770 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.141.3
20:04:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:02.394910 27770 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.141.20:33323
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-2/data
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-2/wal
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.141.3:0
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-2/data/info.pb
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.141.3
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.141.62:36061,127.26.141.61:44531,127.26.141.60:43105
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.141.3
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-2/logs
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:02.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:02.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:02.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:02.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6b1bbe17b8ab938291541389a90c67c358bba716
20:04:02.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:02.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jun 2026 19:43:21 UTC on e7f111948823
20:04:02.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12395
20:04:02.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:02.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:02.396776 27770 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:02.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:02.399466 27770 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:02.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:02.416962 27776 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:02.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:02.960089 27760 heartbeater.cc:499] Master 127.26.141.60:43105 was elected leader, sending a full tablet report...
20:04:03.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:02.423818 27779 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:03.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:02.419651 27777 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:03.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:03.712466 27778 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1289 milliseconds
20:04:03.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.712596 27770 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:03.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.714072 27770 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.717583 27770 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.725359 27770 hybrid_clock.cc:648] HybridClock initialized: now 1781467443723633 us; error 1719 us; skew 500 ppm
20:04:03.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.726917 27770 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:03.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.735522 27770 webserver.cc:533] Webserver started at http://127.26.141.3:33377/ using document root <none> and password file <none>
20:04:03.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.736883 27770 fs_manager.cc:362] Metadata directory not provided
20:04:03.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.737192 27770 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.737884 27770 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:03.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.744964 27770 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-2/data/instance:
20:04:03.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b39ca52d2fc447729f1e3793551a5188"
20:04:03.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:03 on dist-test-slave-h0zs"
20:04:03.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.746639 27770 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-2/wal/instance:
20:04:03.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b39ca52d2fc447729f1e3793551a5188"
20:04:03.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:03 on dist-test-slave-h0zs"
20:04:03.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.756093 27770 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.002s sys 0.008s
20:04:03.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.762382 27787 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:03.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.763589 27770 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
20:04:03.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.763896 27770 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-2/data,/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-2/wal
20:04:03.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b39ca52d2fc447729f1e3793551a5188"
20:04:03.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:03 on dist-test-slave-h0zs"
20:04:03.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.764220 27770 fs_report.cc:389] FS layout report
20:04:03.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:03.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-2/wal
20:04:03.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-2/wal
20:04:03.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-2/data/data
20:04:03.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:03.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:03.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:03.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:03.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:03.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:03.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:03.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:03.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:03.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:03.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:03.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.838052 27770 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:03.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.839962 27770 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:03.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.840400 27770 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.843696 27770 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:03.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.848649 27770 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:04:03.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.848870 27770 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:04:03.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.849083 27770 ts_tablet_manager.cc:616] Registered 0 tablets
20:04:03.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:03.849287 27770 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
20:04:04.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.045948 27770 rpc_server.cc:307] RPC server started. Bound to: 127.26.141.3:39693
20:04:04.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.046149 27899 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.141.3:39693 every 8 connection(s)
20:04:04.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.049037 27770 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-2/data/info.pb
20:04:04.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.049654 27188 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu as pid 27770
20:04:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.050030 27188 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster5325272160008083170/ts-2/wal/instance
20:04:04.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.104291 27901 heartbeater.cc:344] Connected to a master server at 127.26.141.61:44531
20:04:04.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.104898 27901 heartbeater.cc:461] Registering TS with master...
20:04:04.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.104915 27900 heartbeater.cc:344] Connected to a master server at 127.26.141.60:43105
20:04:04.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.105391 27900 heartbeater.cc:461] Registering TS with master...
20:04:04.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.106616 27901 heartbeater.cc:507] Master 127.26.141.61:44531 requested a full tablet report, sending...
20:04:04.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.106850 27900 heartbeater.cc:507] Master 127.26.141.60:43105 requested a full tablet report, sending...
20:04:04.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.108101 27902 heartbeater.cc:344] Connected to a master server at 127.26.141.62:36061
20:04:04.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.108435 27902 heartbeater.cc:461] Registering TS with master...
20:04:04.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.109337 27902 heartbeater.cc:507] Master 127.26.141.62:36061 requested a full tablet report, sending...
20:04:04.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.109942 27388 ts_manager.cc:194] Registered new tserver with Master: b39ca52d2fc447729f1e3793551a5188 (127.26.141.3:39693)
20:04:04.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.112692 27388 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.141.3:47551
20:04:04.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.113034 27308 ts_manager.cc:194] Registered new tserver with Master: b39ca52d2fc447729f1e3793551a5188 (127.26.141.3:39693)
20:04:04.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.114264 27233 ts_manager.cc:194] Registered new tserver with Master: b39ca52d2fc447729f1e3793551a5188 (127.26.141.3:39693)
20:04:04.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.121039 27188 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
20:04:04.135 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:04:04.136 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:04:04.147 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.26.141.62"
port: 36061
}
bound_http_address {
host: "127.26.141.62"
port: 33929
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.26.141.61"
port: 44531
}
bound_http_address {
host: "127.26.141.61"
port: 34429
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.26.141.60"
port: 43105
}
bound_http_address {
host: "127.26.141.60"
port: 40785
}
}
}
20:04:04.150 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:04:04.153 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.26.141.1"
port: 44173
}
bound_http_address {
host: "127.26.141.1"
port: 41399
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.26.141.2"
port: 42225
}
bound_http_address {
host: "127.26.141.2"
port: 43779
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.26.141.3"
port: 39693
}
bound_http_address {
host: "127.26.141.3"
port: 33377
}
}
}
20:04:04.154 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:04:04.342 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
20:04:04.401 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:04:04.543 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskCk90tC/test-tmp
20:04:04.544 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:04:04.544 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskCk90tC/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:04:04.558 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
20:04:04.979 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:04.980 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:04:04.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.984036 27912 mini_oidc.cc:150] Starting JWKS server
20:04:04.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.989156 27912 webserver.cc:533] Webserver started at http://127.0.0.1:41857/ using document root <none> and password file <none>
20:04:04.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.989616 27912 mini_oidc.cc:186] Starting OIDC Discovery server
20:04:04.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:04.992110 27912 webserver.cc:533] Webserver started at http://127.0.0.1:40475/ using document root <none> and password file <none>
20:04:05.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
20:04:05.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/principal' for realm 'KRBTEST.COM',
20:04:05.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
20:04:05.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:05 dist-test-slave-h0zs krb5kdc[27924](info): setting up network...
20:04:05.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
20:04:05.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:05 dist-test-slave-h0zs krb5kdc[27924](info): set up 2 sockets
20:04:05.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:05 dist-test-slave-h0zs krb5kdc[27924](info): commencing operation
20:04:05.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
20:04:05.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:05.116883 27900 heartbeater.cc:499] Master 127.26.141.60:43105 was elected leader, sending a full tablet report...
20:04:07.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:07.165319 27912 mini_kdc.cc:121] Time spent starting KDC: real 2.131s user 0.005s sys 0.021s
20:04:07.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
20:04:07.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:04:07.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
20:04:07.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
20:04:07.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:04:07.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
20:04:07.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
20:04:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:04:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
20:04:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:04:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/test-user.keytab.
20:04:07.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/test-user.keytab.
20:04:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:07 dist-test-slave-h0zs krb5kdc[27924](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467447, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:04:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-14T20:04:07Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:04:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-14T20:04:07Z Disabled control of system clock
20:04:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.66.62@KRBTEST.COM; defaulting to no policy
20:04:07.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:07.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.66.62@KRBTEST.COM" created.
20:04:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.66.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:07.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.66.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:07.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.27.66.62@KRBTEST.COM; defaulting to no policy
20:04:07.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:07.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.27.66.62@KRBTEST.COM" created.
20:04:07.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:07.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.66.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:07.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.66.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:07.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:07.427201 27912 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:07.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:07.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/wal
20:04:07.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/data
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/logs
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/data/info.pb
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:07.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:07.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.66.62:40091
20:04:07.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.66.62
20:04:07.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:07.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab
20:04:07.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.66.62
20:04:07.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:07.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.66.20:44487
20:04:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.66.62:40091,127.27.66.61:34759,127.27.66.60:45873
20:04:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41857/jwks.json
20:04:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:04:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:07.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:07.785334 27943 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:07.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:07.785990 27943 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:07.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:07.786573 27943 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:07.831816 27943 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:04:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:07.832185 27943 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:07.832635 27943 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:07.833000 27943 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:07.833424 27943 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:07.833811 27943 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:07.834091 27943 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41857/jwks.json
20:04:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:07.882031 27943 master_runner.cc:387] Master server non-default flags:
20:04:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.66.20:44487
20:04:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/data
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/wal
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.66.62:40091,127.27.66.61:34759,127.27.66.60:45873
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.66.62
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.66.62:40091
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41857/jwks.json
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/data/info.pb
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.66.62
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/logs
20:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:07.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:07.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:07.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:07.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:07.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6b1bbe17b8ab938291541389a90c67c358bba716
20:04:07.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:07.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jun 2026 19:43:21 UTC on e7f111948823
20:04:07.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12395
20:04:07.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:07.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:07.883592 27943 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:07.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:07.885480 27943 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:07.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:07.900259 27949 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:09.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:09.304714 27948 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 27943
20:04:09.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:09.531642 27943 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.630s user 0.539s sys 1.003s
20:04:09.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:09.531988 27943 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.631s user 0.539s sys 1.003s
20:04:09.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:07.902026 27950 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:09.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:09.534572 27952 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:09.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:09.537268 27951 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1631 milliseconds
20:04:09.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.537286 27943 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:09.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.538786 27943 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:09.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.541254 27943 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:09.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.542603 27943 hybrid_clock.cc:648] HybridClock initialized: now 1781467449542567 us; error 36 us; skew 500 ppm
20:04:09.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.543444 27943 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:09.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:09 dist-test-slave-h0zs krb5kdc[27924](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467449, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.549794 27943 init.cc:377] Logged in from keytab as kudu/127.27.66.62@KRBTEST.COM (short username kudu)
20:04:09.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.554565 27943 webserver.cc:533] Webserver started at http://127.27.66.62:33909/ using document root <none> and password file <none>
20:04:09.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.555425 27943 fs_manager.cc:362] Metadata directory not provided
20:04:09.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.555601 27943 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:09.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.556267 27943 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:09.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.560670 27943 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/data/instance:
20:04:09.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "23a268f069b6459d9c8db89b402cb6d0"
20:04:09.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:09 on dist-test-slave-h0zs"
20:04:09.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.561843 27943 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/wal/instance:
20:04:09.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "23a268f069b6459d9c8db89b402cb6d0"
20:04:09.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:09 on dist-test-slave-h0zs"
20:04:09.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.569577 27943 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
20:04:09.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.574718 27963 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.575805 27943 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.003s
20:04:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.576148 27943 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/data,/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/wal
20:04:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "23a268f069b6459d9c8db89b402cb6d0"
20:04:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:09 on dist-test-slave-h0zs"
20:04:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.576427 27943 fs_report.cc:389] FS layout report
20:04:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/wal
20:04:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/wal
20:04:09.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/data/data
20:04:09.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:09.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:09.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:09.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:09.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:09.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:09.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:09.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:09.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:09.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:09.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:09.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.633468 27943 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:09.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.639997 27943 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.641465 27943 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:09.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.743273 27943 rpc_server.cc:307] RPC server started. Bound to: 127.27.66.62:40091
20:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.743350 28014 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.66.62:40091 every 8 connection(s)
20:04:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.747888 27943 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/data/info.pb
20:04:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.755045 27912 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu as pid 27943
20:04:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.755635 27912 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/wal/instance
20:04:09.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.757400 28015 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:04:09.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.776222 28015 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } has no permanent_uuid. Determining permanent_uuid...
20:04:09.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.66.61@KRBTEST.COM; defaulting to no policy
20:04:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.66.61@KRBTEST.COM" created.
20:04:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:09 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467449, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.62@KRBTEST.COM for kudu/127.27.66.62@KRBTEST.COM
20:04:09.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:09.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.66.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:09.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.66.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:09.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.828163 28015 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } has no permanent_uuid. Determining permanent_uuid...
20:04:09.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.27.66.61@KRBTEST.COM; defaulting to no policy
20:04:09.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:09.833441 27967 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.66.61:34759: connect: Connection refused (error 111)
20:04:09.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:09.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.27.66.61@KRBTEST.COM" created.
20:04:09.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:09.842777 28015 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.61:34759: Network error: Client connection negotiation failed: client connection to 127.27.66.61:34759: connect: Connection refused (error 111)
20:04:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:09.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.66.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:09.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.66.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:09.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.875276 27912 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:09.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:09.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/wal
20:04:09.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/data
20:04:09.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:09.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:09.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:09.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:09.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:09.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:09.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/logs
20:04:09.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/data/info.pb
20:04:09.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:09.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:09.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:09.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:09.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:09.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.66.61:34759
20:04:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.66.61
20:04:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab
20:04:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.66.61
20:04:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:09.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.66.20:44487
20:04:09.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:09.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:09.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:09.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:09.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.66.62:40091,127.27.66.61:34759,127.27.66.60:45873
20:04:09.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:09.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41857/jwks.json
20:04:09.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:04:09.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:09.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.892375 28015 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } attempt: 1
20:04:09.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:09.898557 28015 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.61:34759: Network error: Client connection negotiation failed: client connection to 127.27.66.61:34759: connect: Connection refused (error 111)
20:04:09.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:09.967124 28015 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } attempt: 2
20:04:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:09.972371 28015 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.61:34759: Network error: Client connection negotiation failed: client connection to 127.27.66.61:34759: connect: Connection refused (error 111)
20:04:10.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:10.063882 28015 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } attempt: 3
20:04:10.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:10.070513 28015 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.61:34759: Network error: Client connection negotiation failed: client connection to 127.27.66.61:34759: connect: Connection refused (error 111)
20:04:10.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:10.213963 28015 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } attempt: 4
20:04:10.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:10.219794 28015 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.61:34759: Network error: Client connection negotiation failed: client connection to 127.27.66.61:34759: connect: Connection refused (error 111)
20:04:10.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:10.258809 28024 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:10.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:10.259429 28024 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:10.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:10.260334 28024 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:10.301548 28024 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:04:10.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:10.301803 28024 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:10.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:10.302078 28024 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:10.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:10.302387 28024 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:10.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:10.302615 28024 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:10.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:10.302816 28024 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:10.302969 28024 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41857/jwks.json
20:04:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:10.345708 28024 master_runner.cc:387] Master server non-default flags:
20:04:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.66.20:44487
20:04:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/data
20:04:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/wal
20:04:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:04:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.66.62:40091,127.27.66.61:34759,127.27.66.60:45873
20:04:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab
20:04:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.66.61
20:04:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.66.61:34759
20:04:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41857/jwks.json
20:04:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/data/info.pb
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.66.61
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/logs
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6b1bbe17b8ab938291541389a90c67c358bba716
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jun 2026 19:43:21 UTC on e7f111948823
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12395
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:10.347362 28024 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:10.348464 28024 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:10.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:10.361255 28031 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:10.519757 28015 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } attempt: 5
20:04:10.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:10.527931 28015 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.61:34759: Network error: Client connection negotiation failed: client connection to 127.27.66.61:34759: connect: Connection refused (error 111)
20:04:11.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.075557 28015 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } attempt: 6
20:04:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:11.084496 28015 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.61:34759: Network error: Client connection negotiation failed: client connection to 127.27.66.61:34759: connect: Connection refused (error 111)
20:04:11.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:10.362871 28032 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:11.740612 28034 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:11.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:11.743043 28033 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1376 milliseconds
20:04:11.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.743139 28024 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:11.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.745302 28024 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:11.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.749022 28024 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.750694 28024 hybrid_clock.cc:648] HybridClock initialized: now 1781467451750614 us; error 87 us; skew 500 ppm
20:04:11.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.751605 28024 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:11 dist-test-slave-h0zs krb5kdc[27924](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467451, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.760143 28024 init.cc:377] Logged in from keytab as kudu/127.27.66.61@KRBTEST.COM (short username kudu)
20:04:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.767051 28024 webserver.cc:533] Webserver started at http://127.27.66.61:33003/ using document root <none> and password file <none>
20:04:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.768340 28024 fs_manager.cc:362] Metadata directory not provided
20:04:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.768612 28024 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:11.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.769227 28024 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.774940 28024 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/data/instance:
20:04:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e711b730f1a4307a4049a64e51eecd4"
20:04:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:11 on dist-test-slave-h0zs"
20:04:11.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.776031 28024 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/wal/instance:
20:04:11.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e711b730f1a4307a4049a64e51eecd4"
20:04:11.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:11 on dist-test-slave-h0zs"
20:04:11.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.785370 28024 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.010s sys 0.001s
20:04:11.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.791208 28045 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:11.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.792402 28024 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.792727 28024 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/data,/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/wal
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e711b730f1a4307a4049a64e51eecd4"
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:11 on dist-test-slave-h0zs"
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.793061 28024 fs_report.cc:389] FS layout report
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/wal
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/wal
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/data/data
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.869926 28024 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:11.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.883424 28024 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.884032 28024 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:11.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.988713 28024 rpc_server.cc:307] RPC server started. Bound to: 127.27.66.61:34759
20:04:11.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.988812 28096 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.66.61:34759 every 8 connection(s)
20:04:11.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.991928 28024 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/data/info.pb
20:04:11.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.995136 27912 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu as pid 28024
20:04:11.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:11.995736 27912 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/wal/instance
20:04:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.000605 28097 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:04:12.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.66.60@KRBTEST.COM; defaulting to no policy
20:04:12.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:12.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.66.60@KRBTEST.COM" created.
20:04:12.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.014611 28097 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } has no permanent_uuid. Determining permanent_uuid...
20:04:12.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:12 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467451, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.61@KRBTEST.COM for kudu/127.27.66.62@KRBTEST.COM
20:04:12.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.66.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.66.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.060920 28097 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } has no permanent_uuid. Determining permanent_uuid...
20:04:12.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.27.66.60@KRBTEST.COM; defaulting to no policy
20:04:12.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:12.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.27.66.60@KRBTEST.COM" created.
20:04:12.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:12 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467451, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.61@KRBTEST.COM for kudu/127.27.66.61@KRBTEST.COM
20:04:12.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.091398 28097 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } has no permanent_uuid. Determining permanent_uuid...
20:04:12.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.094216 28047 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.66.60:45873: connect: Connection refused (error 111)
20:04:12.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:12.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.66.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:12.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.66.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:12.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.099272 28097 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.60:45873: Network error: Client connection negotiation failed: client connection to 127.27.66.60:45873: connect: Connection refused (error 111)
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.099884 27912 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/wal
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/data
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/logs
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/data/info.pb
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.66.60:45873
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.66.60
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.66.60
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.66.20:44487
20:04:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:12.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:12.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:12.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:12.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.66.62:40091,127.27.66.61:34759,127.27.66.60:45873
20:04:12.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:12.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41857/jwks.json
20:04:12.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:04:12.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.144940 28015 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } attempt: 7
20:04:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.148873 28097 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } attempt: 1
20:04:12.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.156992 28097 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.60:45873: Network error: Client connection negotiation failed: client connection to 127.27.66.60:45873: connect: Connection refused (error 111)
20:04:12.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:12 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467449, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.62@KRBTEST.COM for kudu/127.27.66.61@KRBTEST.COM
20:04:12.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.174746 28015 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } has no permanent_uuid. Determining permanent_uuid...
20:04:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.181396 28015 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.60:45873: Network error: Client connection negotiation failed: client connection to 127.27.66.60:45873: connect: Connection refused (error 111)
20:04:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.225478 28097 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } attempt: 2
20:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.231769 28097 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.60:45873: Network error: Client connection negotiation failed: client connection to 127.27.66.60:45873: connect: Connection refused (error 111)
20:04:12.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.239830 28015 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } attempt: 1
20:04:12.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.245729 28015 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.60:45873: Network error: Client connection negotiation failed: client connection to 127.27.66.60:45873: connect: Connection refused (error 111)
20:04:12.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.323190 28097 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } attempt: 3
20:04:12.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.327198 28015 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } attempt: 2
20:04:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.328287 28097 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.60:45873: Network error: Client connection negotiation failed: client connection to 127.27.66.60:45873: connect: Connection refused (error 111)
20:04:12.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.332019 28015 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.60:45873: Network error: Client connection negotiation failed: client connection to 127.27.66.60:45873: connect: Connection refused (error 111)
20:04:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.417508 28015 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } attempt: 3
20:04:12.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.422240 28015 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.60:45873: Network error: Client connection negotiation failed: client connection to 127.27.66.60:45873: connect: Connection refused (error 111)
20:04:12.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.471763 28097 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } attempt: 4
20:04:12.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.475584 28107 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:12.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.476374 28107 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:12.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.476661 28097 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.60:45873: Network error: Client connection negotiation failed: client connection to 127.27.66.60:45873: connect: Connection refused (error 111)
20:04:12.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.477414 28107 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:12.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.515818 28107 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:04:12.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.516142 28107 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:12.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.516618 28107 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:12.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.516995 28107 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:12.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.517359 28107 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:12.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.517710 28107 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:12.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.517985 28107 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41857/jwks.json
20:04:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.562179 28107 master_runner.cc:387] Master server non-default flags:
20:04:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.66.20:44487
20:04:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/data
20:04:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/wal
20:04:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:04:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.66.62:40091,127.27.66.61:34759,127.27.66.60:45873
20:04:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab
20:04:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.66.60
20:04:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:12.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:12.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:12.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.66.60:45873
20:04:12.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:12.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:12.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:12.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41857/jwks.json
20:04:12.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:12.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:12.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:12.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:12.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:12.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/data/info.pb
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.66.60
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/logs
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6b1bbe17b8ab938291541389a90c67c358bba716
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jun 2026 19:43:21 UTC on e7f111948823
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12395
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.562649 28015 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } attempt: 4
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.564193 28107 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.565843 28107 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:12.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.568187 28015 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.60:45873: Network error: Client connection negotiation failed: client connection to 127.27.66.60:45873: connect: Connection refused (error 111)
20:04:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.582901 28116 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:12.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.776547 28097 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } attempt: 5
20:04:12.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.788434 28097 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.60:45873: Network error: Client connection negotiation failed: client connection to 127.27.66.60:45873: connect: Connection refused (error 111)
20:04:12.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:12.852473 28015 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } attempt: 5
20:04:12.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.862897 28015 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.60:45873: Network error: Client connection negotiation failed: client connection to 127.27.66.60:45873: connect: Connection refused (error 111)
20:04:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:13.336989 28097 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } attempt: 6
20:04:13.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:13.372938 28097 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.60:45873: Network error: Client connection negotiation failed: client connection to 127.27.66.60:45873: connect: Connection refused (error 111)
20:04:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:13.415541 28015 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } attempt: 6
20:04:13.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:13.451675 28015 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.60:45873: Network error: Client connection negotiation failed: client connection to 127.27.66.60:45873: connect: Connection refused (error 111)
20:04:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:13.984541 28115 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 28107
20:04:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:14.232126 28115 kernel_stack_watchdog.cc:198] Thread 28107 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
20:04:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:04:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:04:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:04:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:04:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:14.232396 28107 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.649s user 0.463s sys 1.040s
20:04:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:12.583345 28117 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:14.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:14.232762 28107 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.650s user 0.463s sys 1.040s
20:04:14.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:14.235813 28119 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:14.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:14.238557 28118 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1652 milliseconds
20:04:14.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.238586 28107 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:14.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.240000 28107 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:14.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.243048 28107 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:14.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.244485 28107 hybrid_clock.cc:648] HybridClock initialized: now 1781467454244450 us; error 41 us; skew 500 ppm
20:04:14.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.245556 28107 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:14.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:14 dist-test-slave-h0zs krb5kdc[27924](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467454, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:14.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.253051 28107 init.cc:377] Logged in from keytab as kudu/127.27.66.60@KRBTEST.COM (short username kudu)
20:04:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.260155 28107 webserver.cc:533] Webserver started at http://127.27.66.60:37759/ using document root <none> and password file <none>
20:04:14.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.261337 28107 fs_manager.cc:362] Metadata directory not provided
20:04:14.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.261597 28107 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:14.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.262306 28107 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:14.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.267117 28107 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/data/instance:
20:04:14.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63d7e494c1c449ccbdf58c42eb024ea1"
20:04:14.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:14 on dist-test-slave-h0zs"
20:04:14.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.268549 28107 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/wal/instance:
20:04:14.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63d7e494c1c449ccbdf58c42eb024ea1"
20:04:14.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:14 on dist-test-slave-h0zs"
20:04:14.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.280288 28107 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.005s sys 0.005s
20:04:14.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.286540 28132 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:14.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.287708 28107 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
20:04:14.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.288134 28107 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/data,/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/wal
20:04:14.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63d7e494c1c449ccbdf58c42eb024ea1"
20:04:14.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:14 on dist-test-slave-h0zs"
20:04:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.288516 28107 fs_report.cc:389] FS layout report
20:04:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/wal
20:04:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/wal
20:04:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/data/data
20:04:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:14.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.357007 28107 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:14.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.363494 28107 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:14.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.363948 28107 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:14.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.433638 28097 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } attempt: 7
20:04:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:14.446458 28097 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.66.60:45873: Network error: Client connection negotiation failed: client connection to 127.27.66.60:45873: connect: Connection refused (error 111)
20:04:14.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.465441 28107 rpc_server.cc:307] RPC server started. Bound to: 127.27.66.60:45873
20:04:14.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.465525 28185 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.66.60:45873 every 8 connection(s)
20:04:14.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.468845 28107 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/data/info.pb
20:04:14.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.475919 27912 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu as pid 28107
20:04:14.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.476404 27912 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/wal/instance
20:04:14.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.477276 28186 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:04:14.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.485339 28015 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } attempt: 7
20:04:14.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.502321 28186 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } has no permanent_uuid. Determining permanent_uuid...
20:04:14.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:14 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467447, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.27.66.62@KRBTEST.COM
20:04:14.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:14 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467449, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.62@KRBTEST.COM for kudu/127.27.66.60@KRBTEST.COM
20:04:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:14 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467454, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.60@KRBTEST.COM for kudu/127.27.66.62@KRBTEST.COM
20:04:14.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:14 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467447, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.27.66.61@KRBTEST.COM
20:04:14.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.560477 28186 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } has no permanent_uuid. Determining permanent_uuid...
20:04:14.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.563902 28015 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0: Bootstrap starting.
20:04:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:14 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467447, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.27.66.60@KRBTEST.COM
20:04:14.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.574970 28015 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0: Neither blocks nor log segments found. Creating new log.
20:04:14.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.577925 28015 log.cc:826] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0: Log is configured to *not* fsync() on all Append() calls
20:04:14.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:14 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467454, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.60@KRBTEST.COM for kudu/127.27.66.61@KRBTEST.COM
20:04:14.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.584450 28015 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0: No bootstrap required, opened a new log
20:04:14.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.606410 28186 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } has no permanent_uuid. Determining permanent_uuid...
20:04:14.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:14 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467454, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.60@KRBTEST.COM for kudu/127.27.66.60@KRBTEST.COM
20:04:14.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.628427 28015 raft_consensus.cc:359] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.629833 28015 raft_consensus.cc:385] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:04:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.630136 28015 raft_consensus.cc:740] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 23a268f069b6459d9c8db89b402cb6d0, State: Initialized, Role: FOLLOWER
20:04:14.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.631189 28015 consensus_queue.cc:260] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.638664 28197 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.639242 28015 sys_catalog.cc:565] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [sys.catalog]: configured and running, proceeding with master startup.
20:04:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.639312 28197 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [sys.catalog]: This master's current role is: FOLLOWER
20:04:14.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.654330 28186 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1: Bootstrap starting.
20:04:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:14.654914 27969 tablet.cc:2404] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:04:14.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.666254 28186 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1: Neither blocks nor log segments found. Creating new log.
20:04:14.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.669945 28186 log.cc:826] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1: Log is configured to *not* fsync() on all Append() calls
20:04:14.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.677557 28186 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1: No bootstrap required, opened a new log
20:04:14.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.679536 28197 raft_consensus.cc:493] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:04:14.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.680248 28197 raft_consensus.cc:515] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:14.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.688257 28197 leader_election.cc:290] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5e711b730f1a4307a4049a64e51eecd4 (127.27.66.61:34759), 63d7e494c1c449ccbdf58c42eb024ea1 (127.27.66.60:45873)
20:04:14.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.715987 28186 raft_consensus.cc:359] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:14.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.716935 28186 raft_consensus.cc:385] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:04:14.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.717260 28186 raft_consensus.cc:740] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 63d7e494c1c449ccbdf58c42eb024ea1, State: Initialized, Role: FOLLOWER
20:04:14.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.718299 28186 consensus_queue.cc:260] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:14.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:14.719141 28209 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:04:14.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:14.719722 28209 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:04:14.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.700937 27943 master_runner.cc:187] Error getting master registration for 127.27.66.61:34759: OK, instance_id { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" instance_seqno: 1781467451963161 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:04:14.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.743907 27943 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:04:14.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.770525 28211 sys_catalog.cc:455] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:14.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.771546 28211 sys_catalog.cc:458] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [sys.catalog]: This master's current role is: FOLLOWER
20:04:14.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.773701 28186 sys_catalog.cc:565] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [sys.catalog]: configured and running, proceeding with master startup.
20:04:14.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.778640 28159 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "23a268f069b6459d9c8db89b402cb6d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" is_pre_election: true
20:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.779706 28159 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 23a268f069b6459d9c8db89b402cb6d0 in term 0.
20:04:14.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.781248 27965 leader_election.cc:304] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 23a268f069b6459d9c8db89b402cb6d0, 63d7e494c1c449ccbdf58c42eb024ea1; no voters:
20:04:14.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.782449 28197 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:04:14.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.782790 28197 raft_consensus.cc:493] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:04:14.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.783082 28197 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 0 FOLLOWER]: Advancing to term 1
20:04:14.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.791529 28197 raft_consensus.cc:515] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:14.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.817909 28159 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "23a268f069b6459d9c8db89b402cb6d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "63d7e494c1c449ccbdf58c42eb024ea1"
20:04:14.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.819581 28159 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [term 0 FOLLOWER]: Advancing to term 1
20:04:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.817082 28197 leader_election.cc:290] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [CANDIDATE]: Term 1 election: Requested vote from peers 5e711b730f1a4307a4049a64e51eecd4 (127.27.66.61:34759), 63d7e494c1c449ccbdf58c42eb024ea1 (127.27.66.60:45873)
20:04:14.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.852339 28159 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 23a268f069b6459d9c8db89b402cb6d0 in term 1.
20:04:14.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.854462 27965 leader_election.cc:304] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 23a268f069b6459d9c8db89b402cb6d0, 63d7e494c1c449ccbdf58c42eb024ea1; no voters:
20:04:14.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.855687 28197 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 1 FOLLOWER]: Leader election won for term 1
20:04:14.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:14.871356 28138 tablet.cc:2404] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:04:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.893340 28197 raft_consensus.cc:697] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 1 LEADER]: Becoming Leader. State: Replica: 23a268f069b6459d9c8db89b402cb6d0, State: Running, Role: LEADER
20:04:14.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.895232 28197 consensus_queue.cc:237] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:14.918540 28223 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:04:14.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.921690 28220 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 23a268f069b6459d9c8db89b402cb6d0. Latest consensus state: current_term: 1 leader_uuid: "23a268f069b6459d9c8db89b402cb6d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:14.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:14.918968 28223 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:04:14.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.927811 28220 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [sys.catalog]: This master's current role is: LEADER
20:04:14.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.946702 28227 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
20:04:14.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.955775 28227 catalog_manager.cc:1519] Initializing Kudu cluster ID...
20:04:14.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.985605 28159 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [term 1 FOLLOWER]: Refusing update from remote peer 23a268f069b6459d9c8db89b402cb6d0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:04:14.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.991189 28220 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:04:14.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.935568 28107 master_runner.cc:187] Error getting master registration for 127.27.66.61:34759: OK, instance_id { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" instance_seqno: 1781467451963161 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:04:14.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:14.993139 28107 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:04:15.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:15.025727 28211 sys_catalog.cc:455] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 23a268f069b6459d9c8db89b402cb6d0. Latest consensus state: current_term: 1 leader_uuid: "23a268f069b6459d9c8db89b402cb6d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:15.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:15.036965 28220 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "23a268f069b6459d9c8db89b402cb6d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:15.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:15.039065 28220 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [sys.catalog]: This master's current role is: LEADER
20:04:15.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:15.039803 28211 sys_catalog.cc:458] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [sys.catalog]: This master's current role is: FOLLOWER
20:04:15.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:15.060328 28211 sys_catalog.cc:455] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "23a268f069b6459d9c8db89b402cb6d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:15.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:15.062727 28227 catalog_manager.cc:1382] Generated new cluster ID: c7fa964344734d38bee73a3a28382aaa
20:04:15.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:15.063082 28227 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
20:04:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:15.063428 28211 sys_catalog.cc:458] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [sys.catalog]: This master's current role is: FOLLOWER
20:04:15.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:15.106931 28227 catalog_manager.cc:1405] Generated new certificate authority record
20:04:15.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:15.113364 28227 catalog_manager.cc:1539] Loading token signing keys...
20:04:15.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:15.146382 28227 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0: Generated new TSK 0
20:04:15.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:15.150813 28227 catalog_manager.cc:1549] Initializing in-progress tserver states...
20:04:15.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.66.1@KRBTEST.COM; defaulting to no policy
20:04:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.66.1@KRBTEST.COM" created.
20:04:15.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.66.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.66.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:15.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.27.66.1@KRBTEST.COM; defaulting to no policy
20:04:15.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:15.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.27.66.1@KRBTEST.COM" created.
20:04:15.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:15.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.66.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:15.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.66.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:15.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:15.315732 27912 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:15.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:15.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-0/wal
20:04:15.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-0/data
20:04:15.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:15.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:15.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:15.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:15.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:15.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:15.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-0/logs
20:04:15.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-0/data/info.pb
20:04:15.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:15.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.66.1:0
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.66.1
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.66.1
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.66.62:40091,127.27.66.61:34759,127.27.66.60:45873
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.66.1
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.66.20:44487
20:04:15.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:15.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:15.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:15.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:15.931548 28223 catalog_manager.cc:1294] Loaded cluster ID: c7fa964344734d38bee73a3a28382aaa
20:04:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:15.932893 28223 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1: loading cluster ID for follower catalog manager: success
20:04:15.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:15.956720 28223 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1: acquiring CA information for follower catalog manager: success
20:04:15.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:15.974193 28223 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:04:15.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:15.989619 28238 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:15.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:15.990549 28238 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:15.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:15.991729 28238 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:16.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:16.068912 28238 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:16.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:16.070839 28238 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.66.1
20:04:16.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.149811 28238 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:16.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:16.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.66.20:44487
20:04:16.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:16.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:16.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-0/data
20:04:16.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-0/wal
20:04:16.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab
20:04:16.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.66.1
20:04:16.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:16.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:16.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.66.1:0
20:04:16.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:16.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:16.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:16.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-0/data/info.pb
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.66.1
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.66.62:40091,127.27.66.61:34759,127.27.66.60:45873
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.66.1
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-0/logs
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6b1bbe17b8ab938291541389a90c67c358bba716
20:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:16.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jun 2026 19:43:21 UTC on e7f111948823
20:04:16.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12395
20:04:16.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:16.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.154688 28238 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:16.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.156427 28238 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:16.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:16.183947 28244 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:16.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:16.243252 27967 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.27.66.61:34759 timed out after 1.559s (SENT) [suppressed 13 similar messages]
20:04:16.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.536931 28097 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } attempt: 8
20:04:16.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:16 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467451, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.61@KRBTEST.COM for kudu/127.27.66.60@KRBTEST.COM
20:04:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.637677 28097 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4: Bootstrap starting.
20:04:16.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.657071 28097 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4: Neither blocks nor log segments found. Creating new log.
20:04:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.671736 28097 log.cc:826] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4: Log is configured to *not* fsync() on all Append() calls
20:04:16.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.706523 28097 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4: No bootstrap required, opened a new log
20:04:16.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.779673 28097 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:16.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.791597 28097 raft_consensus.cc:385] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:04:16.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.792472 28097 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5e711b730f1a4307a4049a64e51eecd4, State: Initialized, Role: FOLLOWER
20:04:16.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.802932 28097 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:16.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.816756 28070 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [term 0 FOLLOWER]: Advancing to term 1
20:04:16.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.820884 28072 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "23a268f069b6459d9c8db89b402cb6d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5e711b730f1a4307a4049a64e51eecd4" is_pre_election: true
20:04:16.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:16.822230 28072 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:55108 (request call id 1) took 2137 ms (client timeout 1559 ms). Trace:
20:04:16.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:16.822641 28072 rpcz_store.cc:269] 0614 20:04:14.685058 (+ 0us) service_pool.cc:167] Inserting onto call queue
20:04:16.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:14.685280 (+ 222us) service_pool.cc:224] Handling call
20:04:16.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:16.822123 (+2136843us) inbound_call.cc:177] Queueing success response
20:04:16.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
20:04:16.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.823985 28071 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "23a268f069b6459d9c8db89b402cb6d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5e711b730f1a4307a4049a64e51eecd4"
20:04:16.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:16.824929 28071 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:55108 (request call id 3) took 2011 ms (client timeout 1721 ms). Trace:
20:04:16.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:16.825315 28071 rpcz_store.cc:269] 0614 20:04:14.814058 (+ 0us) service_pool.cc:167] Inserting onto call queue
20:04:16.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:14.814244 (+ 186us) service_pool.cc:224] Handling call
20:04:16.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:16.824866 (+2010622us) inbound_call.cc:177] Queueing success response
20:04:16.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
20:04:16.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.827648 28068 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "23a268f069b6459d9c8db89b402cb6d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5e711b730f1a4307a4049a64e51eecd4"
20:04:16.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.830132 28069 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "23a268f069b6459d9c8db89b402cb6d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5e711b730f1a4307a4049a64e51eecd4" is_pre_election: true
20:04:16.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.841885 28097 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [sys.catalog]: configured and running, proceeding with master startup.
20:04:16.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.850615 28070 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [term 1 FOLLOWER]: Refusing update from remote peer 23a268f069b6459d9c8db89b402cb6d0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:04:16.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.854000 28070 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:55108 (request call id 4) took 1857 ms. Trace:
20:04:16.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.854971 28070 rpcz_store.cc:276] 0614 20:04:14.996732 (+ 0us) service_pool.cc:167] Inserting onto call queue
20:04:16.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:14.997283 (+ 551us) service_pool.cc:224] Handling call
20:04:16.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:16.816334 (+1819051us) raft_consensus.cc:1471] Updating replica for 0 ops
20:04:16.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:16.853430 (+ 37096us) raft_consensus.cc:1697] Filling consensus response to leader.
20:04:16.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:16.853901 (+ 471us) inbound_call.cc:177] Queueing success response
20:04:16.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
20:04:16.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.859294 28220 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:04:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.867699 28255 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:16.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:16.874097 28255 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [sys.catalog]: This master's current role is: FOLLOWER
20:04:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:16.898700 28051 tablet.cc:2404] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:04:17.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:17.027884 28257 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "23a268f069b6459d9c8db89b402cb6d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:17.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:17.027674 28255 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 23a268f069b6459d9c8db89b402cb6d0. Latest consensus state: current_term: 1 leader_uuid: "23a268f069b6459d9c8db89b402cb6d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:17.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:17.028708 28257 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [sys.catalog]: This master's current role is: FOLLOWER
20:04:17.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:17.028795 28255 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [sys.catalog]: This master's current role is: FOLLOWER
20:04:17.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:17.050027 28272 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:17.471719 28220 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "23a268f069b6459d9c8db89b402cb6d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:17.473798 28220 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [sys.catalog]: This master's current role is: LEADER
20:04:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.096004 28272 catalog_manager.cc:1294] Loaded cluster ID: c7fa964344734d38bee73a3a28382aaa
20:04:18.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.097319 28272 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4: loading cluster ID for follower catalog manager: success
20:04:18.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.114073 28272 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4: acquiring CA information for follower catalog manager: success
20:04:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.134479 28272 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:04:18.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:16.208258 28247 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:18.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:18.764559 28246 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2549 milliseconds
20:04:18.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.764685 28238 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:18.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:16.205643 28245 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:18.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.767130 28238 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:18.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.772461 28238 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:18.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.774111 28238 hybrid_clock.cc:648] HybridClock initialized: now 1781467458774066 us; error 78 us; skew 500 ppm
20:04:18.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.775411 28238 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:18.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:18 dist-test-slave-h0zs krb5kdc[27924](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467458, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:18.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.801047 28238 init.cc:377] Logged in from keytab as kudu/127.27.66.1@KRBTEST.COM (short username kudu)
20:04:18.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.849002 28238 webserver.cc:533] Webserver started at http://127.27.66.1:37415/ using document root <none> and password file <none>
20:04:18.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.851159 28238 fs_manager.cc:362] Metadata directory not provided
20:04:18.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.851841 28238 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:18.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.853063 28238 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:18.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.866561 28238 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-0/data/instance:
20:04:18.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "46c5297fad4b4e75b309237170619903"
20:04:18.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:18 on dist-test-slave-h0zs"
20:04:18.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.894143 28238 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-0/wal/instance:
20:04:18.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "46c5297fad4b4e75b309237170619903"
20:04:18.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:18 on dist-test-slave-h0zs"
20:04:18.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.927637 28238 fs_manager.cc:696] Time spent creating directory manager: real 0.029s user 0.012s sys 0.000s
20:04:18.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.946026 28287 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:18.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.948355 28238 fs_manager.cc:730] Time spent opening block manager: real 0.015s user 0.002s sys 0.008s
20:04:18.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.951085 28238 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-0/data,/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-0/wal
20:04:18.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "46c5297fad4b4e75b309237170619903"
20:04:18.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:18 on dist-test-slave-h0zs"
20:04:18.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:18.952145 28238 fs_report.cc:389] FS layout report
20:04:18.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:18.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-0/wal
20:04:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-0/wal
20:04:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-0/data/data
20:04:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:19.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.125312 28238 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:19.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.140691 28238 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:19.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.141356 28238 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:19.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.149955 28238 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:19.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.155478 28238 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:04:19.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.155722 28238 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:04:19.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.156039 28238 ts_tablet_manager.cc:616] Registered 0 tablets
20:04:19.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.156265 28238 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:04:19.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.460350 28238 rpc_server.cc:307] RPC server started. Bound to: 127.27.66.1:38573
20:04:19.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.461243 28399 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.66.1:38573 every 8 connection(s)
20:04:19.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.464254 28238 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-0/data/info.pb
20:04:19.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.465376 27912 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu as pid 28238
20:04:19.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.465907 27912 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-0/wal/instance
20:04:19.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.66.2@KRBTEST.COM; defaulting to no policy
20:04:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.66.2@KRBTEST.COM" created.
20:04:19.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:19 dist-test-slave-h0zs krb5kdc[27924](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467459, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:19.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.556280 27959 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:19.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:19 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467458, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.1@KRBTEST.COM for kudu/127.27.66.60@KRBTEST.COM
20:04:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:19 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467458, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.1@KRBTEST.COM for kudu/127.27.66.61@KRBTEST.COM
20:04:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.599867 28400 heartbeater.cc:344] Connected to a master server at 127.27.66.60:45873
20:04:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.600494 28400 heartbeater.cc:461] Registering TS with master...
20:04:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.602243 28400 heartbeater.cc:507] Master 127.27.66.60:45873 requested a full tablet report, sending...
20:04:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:19 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467458, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.1@KRBTEST.COM for kudu/127.27.66.62@KRBTEST.COM
20:04:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.607242 28149 ts_manager.cc:194] Registered new tserver with Master: 46c5297fad4b4e75b309237170619903 (127.27.66.1:38573)
20:04:19.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.611153 28402 heartbeater.cc:344] Connected to a master server at 127.27.66.61:34759
20:04:19.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.612557 28402 heartbeater.cc:461] Registering TS with master...
20:04:19.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.613690 28402 heartbeater.cc:507] Master 127.27.66.61:34759 requested a full tablet report, sending...
20:04:19.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:19.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.66.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:19.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.66.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:19.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:19.617462 27912 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.27.66.2: real 0.132s user 0.003s sys 0.016s
20:04:19.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.617935 28062 ts_manager.cc:194] Registered new tserver with Master: 46c5297fad4b4e75b309237170619903 (127.27.66.1:38573)
20:04:19.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.633474 28405 heartbeater.cc:344] Connected to a master server at 127.27.66.62:40091
20:04:19.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.633867 28405 heartbeater.cc:461] Registering TS with master...
20:04:19.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.634824 28405 heartbeater.cc:507] Master 127.27.66.62:40091 requested a full tablet report, sending...
20:04:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.637691 27979 ts_manager.cc:194] Registered new tserver with Master: 46c5297fad4b4e75b309237170619903 (127.27.66.1:38573)
20:04:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.27.66.2@KRBTEST.COM; defaulting to no policy
20:04:19.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.640215 27979 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.27.66.1@KRBTEST.COM'} at 127.27.66.1:37975
20:04:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.27.66.2@KRBTEST.COM" created.
20:04:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:19.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.66.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:19.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.66.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:19.670826 27912 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-1/wal
20:04:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-1/data
20:04:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-1/logs
20:04:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-1/data/info.pb
20:04:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.66.2:0
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.66.2
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.66.2
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.66.62:40091,127.27.66.61:34759,127.27.66.60:45873
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.66.2
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.66.20:44487
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:20.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:20.167479 28414 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:20.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:20.168303 28414 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:20.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:20.169031 28414 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:20.209568 28414 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:20.210598 28414 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.66.2
20:04:20.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:20.270850 28414 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:20.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:20.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.66.20:44487
20:04:20.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-1/data
20:04:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-1/wal
20:04:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab
20:04:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.66.2
20:04:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:20.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.66.2:0
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-1/data/info.pb
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.66.2
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.66.62:40091,127.27.66.61:34759,127.27.66.60:45873
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.66.2
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-1/logs
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6b1bbe17b8ab938291541389a90c67c358bba716
20:04:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jun 2026 19:43:21 UTC on e7f111948823
20:04:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12395
20:04:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:20.273155 28414 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:20.274775 28414 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:20.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:20.293159 28422 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:20.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:20.646773 28405 heartbeater.cc:499] Master 127.27.66.62:40091 was elected leader, sending a full tablet report...
20:04:21.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:21 dist-test-slave-h0zs krb5kdc[27924](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467461, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:21.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:21.766330 28041 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:21.692819 28421 debug-util.cc:398] Leaking SignalData structure 0x7b08000370e0 after lost signal to thread 28414
20:04:22.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:22.001215 28414 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.710s user 0.538s sys 1.049s
20:04:22.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:22.001593 28414 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.710s user 0.538s sys 1.049s
20:04:22.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:20.293919 28423 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:22.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:22.013937 28431 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:22.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.013954 28414 server_base.cc:1061] running on GCE node
20:04:22.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.016384 28414 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.019376 28414 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:22.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.020862 28414 hybrid_clock.cc:648] HybridClock initialized: now 1781467462020680 us; error 189 us; skew 500 ppm
20:04:22.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.022024 28414 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:22.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:22 dist-test-slave-h0zs krb5kdc[27924](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467462, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.030999 28414 init.cc:377] Logged in from keytab as kudu/127.27.66.2@KRBTEST.COM (short username kudu)
20:04:22.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.038842 28414 webserver.cc:533] Webserver started at http://127.27.66.2:40343/ using document root <none> and password file <none>
20:04:22.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.040165 28414 fs_manager.cc:362] Metadata directory not provided
20:04:22.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.040468 28414 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:22.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.041059 28414 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:22.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.048254 28414 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-1/data/instance:
20:04:22.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c6724f14c8514542a61de931d0e8cb91"
20:04:22.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:22 on dist-test-slave-h0zs"
20:04:22.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.050031 28414 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-1/wal/instance:
20:04:22.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c6724f14c8514542a61de931d0e8cb91"
20:04:22.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:22 on dist-test-slave-h0zs"
20:04:22.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.061550 28414 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.010s sys 0.000s
20:04:22.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.068112 28438 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:22.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.069706 28414 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
20:04:22.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.070109 28414 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-1/data,/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-1/wal
20:04:22.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c6724f14c8514542a61de931d0e8cb91"
20:04:22.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:22 on dist-test-slave-h0zs"
20:04:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.070578 28414 fs_report.cc:389] FS layout report
20:04:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-1/wal
20:04:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-1/wal
20:04:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-1/data/data
20:04:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:22.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.155051 28414 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:22.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.161579 28414 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:22.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.162106 28414 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.165596 28414 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.170353 28414 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:04:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.170557 28414 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:04:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.170750 28414 ts_tablet_manager.cc:616] Registered 0 tablets
20:04:22.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.170913 28414 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:04:22.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.368260 28414 rpc_server.cc:307] RPC server started. Bound to: 127.27.66.2:44429
20:04:22.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.368453 28551 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.66.2:44429 every 8 connection(s)
20:04:22.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.371449 28414 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-1/data/info.pb
20:04:22.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.378497 27912 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu as pid 28414
20:04:22.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.379025 27912 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-1/wal/instance
20:04:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.66.3@KRBTEST.COM; defaulting to no policy
20:04:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:22 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467462, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.2@KRBTEST.COM for kudu/127.27.66.60@KRBTEST.COM
20:04:22.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:22.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.66.3@KRBTEST.COM" created.
20:04:22.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:22 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467462, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.2@KRBTEST.COM for kudu/127.27.66.61@KRBTEST.COM
20:04:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:22 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467462, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.2@KRBTEST.COM for kudu/127.27.66.62@KRBTEST.COM
20:04:22.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.484807 28553 heartbeater.cc:344] Connected to a master server at 127.27.66.61:34759
20:04:22.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.485412 28553 heartbeater.cc:461] Registering TS with master...
20:04:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.487005 28553 heartbeater.cc:507] Master 127.27.66.61:34759 requested a full tablet report, sending...
20:04:22.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.490900 28062 ts_manager.cc:194] Registered new tserver with Master: c6724f14c8514542a61de931d0e8cb91 (127.27.66.2:44429)
20:04:22.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.492259 28554 heartbeater.cc:344] Connected to a master server at 127.27.66.62:40091
20:04:22.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.492674 28554 heartbeater.cc:461] Registering TS with master...
20:04:22.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.493891 28554 heartbeater.cc:507] Master 127.27.66.62:40091 requested a full tablet report, sending...
20:04:22.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.496837 28552 heartbeater.cc:344] Connected to a master server at 127.27.66.60:45873
20:04:22.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.496826 27979 ts_manager.cc:194] Registered new tserver with Master: c6724f14c8514542a61de931d0e8cb91 (127.27.66.2:44429)
20:04:22.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.497318 28552 heartbeater.cc:461] Registering TS with master...
20:04:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.498374 28552 heartbeater.cc:507] Master 127.27.66.60:45873 requested a full tablet report, sending...
20:04:22.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.499280 27979 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.27.66.2@KRBTEST.COM'} at 127.27.66.2:56217
20:04:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.66.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:22.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.66.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:22.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.504719 28149 ts_manager.cc:194] Registered new tserver with Master: c6724f14c8514542a61de931d0e8cb91 (127.27.66.2:44429)
20:04:22.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:22.504755 27912 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.27.66.3: real 0.101s user 0.001s sys 0.018s
20:04:22.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.27.66.3@KRBTEST.COM; defaulting to no policy
20:04:22.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:22.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.27.66.3@KRBTEST.COM" created.
20:04:22.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:22.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.66.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:22.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.66.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab.
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:22.553850 27912 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-2/wal
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-2/data
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-2/logs
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-2/data/info.pb
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.66.3:0
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.66.3
20:04:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.66.3
20:04:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.66.62:40091,127.27.66.61:34759,127.27.66.60:45873
20:04:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab
20:04:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.66.3
20:04:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.66.20:44487
20:04:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:22.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:22.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:22.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:22.573067 27614 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9540 after lost signal to thread 27488
20:04:22.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:22.574285 27614 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9920 after lost signal to thread 27617
20:04:22.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:22.605057 28093 debug-util.cc:398] Leaking SignalData structure 0x7b08000712e0 after lost signal to thread 28026
20:04:23.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:23.041740 28567 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:23.043125 28567 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:23.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:23.044288 28567 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:23.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:23.126484 28567 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:23.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:23.128376 28567 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.66.3
20:04:23.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:23.213927 28567 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:23.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:23.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.66.20:44487
20:04:23.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-2/data
20:04:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-2/wal
20:04:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab
20:04:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.66.3
20:04:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.66.3:0
20:04:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-2/data/info.pb
20:04:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.66.3
20:04:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.66.62:40091,127.27.66.61:34759,127.27.66.60:45873
20:04:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.66.3
20:04:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-2/logs
20:04:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6b1bbe17b8ab938291541389a90c67c358bba716
20:04:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jun 2026 19:43:21 UTC on e7f111948823
20:04:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12395
20:04:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:23.216316 28567 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:23.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:23.218180 28567 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:23.279690 28574 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:23.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:23.504004 28554 heartbeater.cc:499] Master 127.27.66.62:40091 was elected leader, sending a full tablet report...
20:04:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:24 dist-test-slave-h0zs krb5kdc[27924](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467464, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:24.260710 28128 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:26.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:26.957671 28567 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.673s user 0.000s sys 0.002s
20:04:26.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:26.959049 28567 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.674s user 0.000s sys 0.003s
20:04:26.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:26.959455 28575 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 3676 milliseconds
20:04:26.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:24.659989 28572 debug-util.cc:398] Leaking SignalData structure 0x7b08000370e0 after lost signal to thread 28567
20:04:26.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:26.959914 28567 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:26.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:26.960615 28578 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:26.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:23.281277 28573 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:26.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:26.964826 28567 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:26.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:26.976665 28567 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:26.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:26.978521 28567 hybrid_clock.cc:648] HybridClock initialized: now 1781467466978423 us; error 99 us; skew 500 ppm
20:04:26.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:26.979737 28567 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:26.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:26 dist-test-slave-h0zs krb5kdc[27924](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467466, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:26.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:26.986788 28567 init.cc:377] Logged in from keytab as kudu/127.27.66.3@KRBTEST.COM (short username kudu)
20:04:26.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:26.993160 28567 webserver.cc:533] Webserver started at http://127.27.66.3:34021/ using document root <none> and password file <none>
20:04:26.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:26.994431 28567 fs_manager.cc:362] Metadata directory not provided
20:04:26.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:26.994858 28567 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:26.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:26.995455 28567 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:27.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.002951 28567 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-2/data/instance:
20:04:27.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33d0aab0850a4967a59f650971b7ae40"
20:04:27.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:26 on dist-test-slave-h0zs"
20:04:27.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.004534 28567 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-2/wal/instance:
20:04:27.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33d0aab0850a4967a59f650971b7ae40"
20:04:27.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:26 on dist-test-slave-h0zs"
20:04:27.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.014648 28567 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.007s sys 0.001s
20:04:27.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.023128 28589 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:27.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.024531 28567 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.004s
20:04:27.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.024935 28567 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-2/data,/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-2/wal
20:04:27.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33d0aab0850a4967a59f650971b7ae40"
20:04:27.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:26 on dist-test-slave-h0zs"
20:04:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.025640 28567 fs_report.cc:389] FS layout report
20:04:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-2/wal
20:04:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-2/wal
20:04:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-2/data/data
20:04:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:27.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.104182 28567 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:27.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.112474 28567 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:27.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.113098 28567 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:27.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.118566 28567 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:27.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.130522 28567 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:04:27.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.130872 28567 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:04:27.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.131181 28567 ts_tablet_manager.cc:616] Registered 0 tablets
20:04:27.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.131397 28567 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:04:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:27.169260 28396 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9be0 after lost signal to thread 28239
20:04:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:27.171022 28396 debug-util.cc:398] Leaking SignalData structure 0x7b08000adb00 after lost signal to thread 28399
20:04:27.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.405843 28567 rpc_server.cc:307] RPC server started. Bound to: 127.27.66.3:38985
20:04:27.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.405972 28701 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.66.3:38985 every 8 connection(s)
20:04:27.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.409044 28567 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-2/data/info.pb
20:04:27.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.413379 27912 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu as pid 28567
20:04:27.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.414132 27912 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/ts-2/wal/instance
20:04:27.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:27 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467466, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.3@KRBTEST.COM for kudu/127.27.66.60@KRBTEST.COM
20:04:27.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:27 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467466, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.3@KRBTEST.COM for kudu/127.27.66.62@KRBTEST.COM
20:04:27.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:27 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467466, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.3@KRBTEST.COM for kudu/127.27.66.61@KRBTEST.COM
20:04:27.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.601541 28706 heartbeater.cc:344] Connected to a master server at 127.27.66.62:40091
20:04:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.602299 28706 heartbeater.cc:461] Registering TS with master...
20:04:27.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.605108 28706 heartbeater.cc:507] Master 127.27.66.62:40091 requested a full tablet report, sending...
20:04:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.613354 27979 ts_manager.cc:194] Registered new tserver with Master: 33d0aab0850a4967a59f650971b7ae40 (127.27.66.3:38985)
20:04:27.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.616278 27979 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.27.66.3@KRBTEST.COM'} at 127.27.66.3:42905
20:04:27.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.632398 28702 heartbeater.cc:344] Connected to a master server at 127.27.66.60:45873
20:04:27.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.632984 28702 heartbeater.cc:461] Registering TS with master...
20:04:27.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.634228 28702 heartbeater.cc:507] Master 127.27.66.60:45873 requested a full tablet report, sending...
20:04:27.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.639110 28149 ts_manager.cc:194] Registered new tserver with Master: 33d0aab0850a4967a59f650971b7ae40 (127.27.66.3:38985)
20:04:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.641631 28703 heartbeater.cc:344] Connected to a master server at 127.27.66.61:34759
20:04:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.642131 28703 heartbeater.cc:461] Registering TS with master...
20:04:27.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.643203 28703 heartbeater.cc:507] Master 127.27.66.61:34759 requested a full tablet report, sending...
20:04:27.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.647655 28062 ts_manager.cc:194] Registered new tserver with Master: 33d0aab0850a4967a59f650971b7ae40 (127.27.66.3:38985)
20:04:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.662592 27912 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
20:04:27.696 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:27.697 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
20:04:27.708 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
20:04:27.710 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
20:04:27.718 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.27.66.62"
port: 40091
}
bound_http_address {
host: "127.27.66.62"
port: 33909
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.27.66.61"
port: 34759
}
bound_http_address {
host: "127.27.66.61"
port: 33003
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.27.66.60"
port: 45873
}
bound_http_address {
host: "127.27.66.60"
port: 37759
}
}
}
20:04:27.718 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:04:27.721 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.27.66.1"
port: 38573
}
bound_http_address {
host: "127.27.66.1"
port: 37415
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.27.66.2"
port: 44429
}
bound_http_address {
host: "127.27.66.2"
port: 40343
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.27.66.3"
port: 38985
}
bound_http_address {
host: "127.27.66.3"
port: 34021
}
}
}
20:04:27.722 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:04:27.724 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:04:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:27 dist-test-slave-h0zs krb5kdc[27924](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467467, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:04:27.786 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:27.789 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5cc
20:04:27.798 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:04:27.799 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5cc
20:04:27.801 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:04:27.827 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
20:04:27.829 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:04:27.829 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:04:27.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:27 dist-test-slave-h0zs krb5kdc[27924](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467467, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:27.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:04:27.894 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:27.894 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5cc
20:04:27.896 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:04:27.896 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.66.62:40091
20:04:27.903 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:04:27.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.903995 27912 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu with pid 27943
20:04:27.947 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:27.947 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.66.60:45873
20:04:27.947 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:04:27.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.948307 27912 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu with pid 28107
20:04:27.990 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:27.990 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.66.61:34759
20:04:27.990 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:04:27.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:27.991252 27912 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu with pid 28024
20:04:28.030 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:28.030 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.27.66.62:40091
20:04:28.035 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:04:28.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:28.037810 27912 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:28.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:28.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/wal
20:04:28.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/data
20:04:28.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:28.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:28.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:28.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:28.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:28.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:28.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/logs
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/data/info.pb
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.66.62:40091
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.66.62
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33909
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.66.62
20:04:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.66.20:44487
20:04:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.66.62:40091,127.27.66.61:34759,127.27.66.60:45873
20:04:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41857/jwks.json
20:04:28.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:04:28.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:28.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:28.413645 28717 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:28.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:28.414522 28717 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:28.415627 28717 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:28.454969 28717 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:04:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:28.455299 28717 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:28.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:28.455811 28717 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:28.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:28.456187 28717 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:28.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:28.456548 28717 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:28.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:28.456899 28717 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:28.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:28.457183 28717 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41857/jwks.json
20:04:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:28.503484 28717 master_runner.cc:387] Master server non-default flags:
20:04:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.66.20:44487
20:04:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/data
20:04:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/wal
20:04:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:04:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.66.62:40091,127.27.66.61:34759,127.27.66.60:45873
20:04:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab
20:04:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.66.62
20:04:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:28.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.66.62:40091
20:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41857/jwks.json
20:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/data/info.pb
20:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.66.62
20:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33909
20:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/logs
20:04:28.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:28.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:28.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:28.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:28.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:28.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6b1bbe17b8ab938291541389a90c67c358bba716
20:04:28.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:28.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jun 2026 19:43:21 UTC on e7f111948823
20:04:28.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12395
20:04:28.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:28.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:28.505558 28717 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:28.507946 28717 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:28.523849 28724 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:28.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:28.551676 28553 heartbeater.cc:646] Failed to heartbeat to 127.27.66.61:34759 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.27.66.61:34759: connect: Connection refused (error 111)
20:04:28.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:28.624943 28706 heartbeater.cc:499] Master 127.27.66.62:40091 was elected leader, sending a full tablet report...
20:04:28.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:28.633378 28706 heartbeater.cc:646] Failed to heartbeat to 127.27.66.62:40091 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.27.66.62:40091: connect: Connection refused (error 111)
20:04:28.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:28.697023 28402 heartbeater.cc:646] Failed to heartbeat to 127.27.66.61:34759 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.27.66.61:34759: connect: Connection refused (error 111)
20:04:28.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:28 dist-test-slave-h0zs krb5kdc[27924](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467468, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:28.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:28.815304 28283 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:30.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:30.101388 27264 debug-util.cc:398] Leaking SignalData structure 0x7b08000a1ea0 after lost signal to thread 27201
20:04:30.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:30.102614 27264 debug-util.cc:398] Leaking SignalData structure 0x7b08000a7780 after lost signal to thread 27267
20:04:31.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:29.927275 28723 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 28717
20:04:31.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:31.045979 28723 kernel_stack_watchdog.cc:198] Thread 28717 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
20:04:31.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:04:31.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:04:31.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:04:31.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:04:31.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:28.524729 28725 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:31.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:31.050812 28717 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.526s user 0.739s sys 1.211s
20:04:31.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:31.051199 28717 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.526s user 0.739s sys 1.211s
20:04:31.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:31.053009 28727 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:31.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:31.055100 28726 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2527 milliseconds
20:04:31.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.055615 28717 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:31.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.057211 28717 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:31.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.060559 28717 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:31.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.062013 28717 hybrid_clock.cc:648] HybridClock initialized: now 1781467471061896 us; error 116 us; skew 500 ppm
20:04:31.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.063153 28717 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:31.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:31 dist-test-slave-h0zs krb5kdc[27924](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467471, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:31.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.071408 28717 init.cc:377] Logged in from keytab as kudu/127.27.66.62@KRBTEST.COM (short username kudu)
20:04:31.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.079119 28717 webserver.cc:533] Webserver started at http://127.27.66.62:33909/ using document root <none> and password file <none>
20:04:31.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.080440 28717 fs_manager.cc:362] Metadata directory not provided
20:04:31.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.080729 28717 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.089627 28717 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.001s
20:04:31.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.094417 28763 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:31.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.095798 28717 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
20:04:31.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.096175 28717 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/data,/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/wal
20:04:31.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "23a268f069b6459d9c8db89b402cb6d0"
20:04:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:09 on dist-test-slave-h0zs"
20:04:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.098191 28717 fs_report.cc:389] FS layout report
20:04:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/wal
20:04:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/wal
20:04:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/data/data
20:04:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.171149 28717 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:31.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.186219 28717 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:31.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.186791 28717 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:31.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.298372 28717 rpc_server.cc:307] RPC server started. Bound to: 127.27.66.62:40091
20:04:31.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.298445 28815 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.66.62:40091 every 8 connection(s)
20:04:31.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.301776 28717 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-0/data/info.pb
20:04:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.305371 27912 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu as pid 28717
20:04:31.306 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:31.306 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.27.66.60:45873
20:04:31.307 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:04:31.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.309021 27912 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:31.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:31.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/wal
20:04:31.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/data
20:04:31.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:31.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:31.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:31.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/logs
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/data/info.pb
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.66.60:45873
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.66.60
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37759
20:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab
20:04:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.66.60
20:04:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.66.20:44487
20:04:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:31.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.66.62:40091,127.27.66.61:34759,127.27.66.60:45873
20:04:31.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:31.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41857/jwks.json
20:04:31.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:04:31.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:31.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.321298 28816 sys_catalog.cc:263] Verifying existing consensus state
20:04:31.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.331849 28816 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0: Bootstrap starting.
20:04:31.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.393954 28816 log.cc:826] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0: Log is configured to *not* fsync() on all Append() calls
20:04:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.416206 28816 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.417375 28816 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0: Bootstrap complete.
20:04:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.457603 28816 raft_consensus.cc:359] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:31.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.461730 28816 raft_consensus.cc:740] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 23a268f069b6459d9c8db89b402cb6d0, State: Initialized, Role: FOLLOWER
20:04:31.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.463145 28816 consensus_queue.cc:260] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:31.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.466799 28820 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.468029 28820 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [sys.catalog]: This master's current role is: FOLLOWER
20:04:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.468954 28816 sys_catalog.cc:565] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [sys.catalog]: configured and running, proceeding with master startup.
20:04:31.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.533789 28831 catalog_manager.cc:1294] Loaded cluster ID: c7fa964344734d38bee73a3a28382aaa
20:04:31.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.534092 28831 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0: loading cluster ID for follower catalog manager: success
20:04:31.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.544255 28831 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0: acquiring CA information for follower catalog manager: success
20:04:31.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.550789 28831 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:04:31.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:31 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467471, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.62@KRBTEST.COM for kudu/127.27.66.62@KRBTEST.COM
20:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.586257 28717 master_runner.cc:187] Error getting master registration for 127.27.66.61:34759: Network error: Client connection negotiation failed: client connection to 127.27.66.61:34759: connect: Connection refused (error 111),
20:04:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.620055 28717 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:04:31.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.653024 28554 heartbeater.cc:344] Connected to a master server at 127.27.66.62:40091
20:04:31.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.660475 28780 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c6724f14c8514542a61de931d0e8cb91" instance_seqno: 1781467462323475) as {username='kudu', principal='kudu/127.27.66.2@KRBTEST.COM'} at 127.27.66.2:60889; Asking this server to re-register.
20:04:31.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.662951 28554 heartbeater.cc:461] Registering TS with master...
20:04:31.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.667524 28779 ts_manager.cc:194] Registered new tserver with Master: c6724f14c8514542a61de931d0e8cb91 (127.27.66.2:44429)
20:04:31.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.727432 28706 heartbeater.cc:344] Connected to a master server at 127.27.66.62:40091
20:04:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.727973 28706 heartbeater.cc:499] Master 127.27.66.62:40091 was elected leader, sending a full tablet report...
20:04:31.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.732132 28780 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "33d0aab0850a4967a59f650971b7ae40" instance_seqno: 1781467467344730) as {username='kudu', principal='kudu/127.27.66.3@KRBTEST.COM'} at 127.27.66.3:36897; Asking this server to re-register.
20:04:31.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.735140 28706 heartbeater.cc:461] Registering TS with master...
20:04:31.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.739050 28780 ts_manager.cc:194] Registered new tserver with Master: 33d0aab0850a4967a59f650971b7ae40 (127.27.66.3:38985)
20:04:31.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.775696 28405 heartbeater.cc:344] Connected to a master server at 127.27.66.62:40091
20:04:31.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.779536 28780 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "46c5297fad4b4e75b309237170619903" instance_seqno: 1781467459404385) as {username='kudu', principal='kudu/127.27.66.1@KRBTEST.COM'} at 127.27.66.1:33743; Asking this server to re-register.
20:04:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.781529 28405 heartbeater.cc:461] Registering TS with master...
20:04:31.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:31.786478 28780 ts_manager.cc:194] Registered new tserver with Master: 46c5297fad4b4e75b309237170619903 (127.27.66.1:38573)
20:04:31.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:31.938830 28818 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:31.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:31.939747 28818 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:31.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:31.940858 28818 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:32.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:32.008224 28818 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:04:32.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:32.008595 28818 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:32.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:32.009073 28818 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:32.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:32.009450 28818 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:32.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:32.009799 28818 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:32.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:32.010121 28818 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:32.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:32.010380 28818 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41857/jwks.json
20:04:32.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:32 dist-test-slave-h0zs krb5kdc[27924](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467472, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:32.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:32.038311 28434 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:32.078935 28818 master_runner.cc:387] Master server non-default flags:
20:04:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.66.20:44487
20:04:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/data
20:04:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/wal
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.66.62:40091,127.27.66.61:34759,127.27.66.60:45873
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.66.60
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.66.60:45873
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41857/jwks.json
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/data/info.pb
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.66.60
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37759
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:32.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:32.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:32.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/logs
20:04:32.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:32.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:32.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:32.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:32.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:32.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6b1bbe17b8ab938291541389a90c67c358bba716
20:04:32.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:32.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jun 2026 19:43:21 UTC on e7f111948823
20:04:32.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12395
20:04:32.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:32.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:32.081153 28818 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:32.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:32.082854 28818 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:32.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:32.100481 28848 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:33.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:33.283334 27756 debug-util.cc:398] Leaking SignalData structure 0x7b08000b56e0 after lost signal to thread 27629
20:04:33.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:33.284865 27756 debug-util.cc:398] Leaking SignalData structure 0x7b08000ade00 after lost signal to thread 27759
20:04:33.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:33.344836 28862 raft_consensus.cc:493] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:04:33.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:33.348665 28862 raft_consensus.cc:515] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:33.394418 28765 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.66.60:45873: connect: Connection refused (error 111)
20:04:33.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:33.398717 28862 leader_election.cc:290] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5e711b730f1a4307a4049a64e51eecd4 (127.27.66.61:34759), 63d7e494c1c449ccbdf58c42eb024ea1 (127.27.66.60:45873)
20:04:33.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:33.409663 28767 leader_election.cc:336] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 5e711b730f1a4307a4049a64e51eecd4 (127.27.66.61:34759): Network error: Client connection negotiation failed: client connection to 127.27.66.61:34759: connect: Connection refused (error 111)
20:04:33.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:33.418499 28765 leader_election.cc:336] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 63d7e494c1c449ccbdf58c42eb024ea1 (127.27.66.60:45873): Network error: Client connection negotiation failed: client connection to 127.27.66.60:45873: connect: Connection refused (error 111)
20:04:33.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:33.419641 28765 leader_election.cc:304] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 23a268f069b6459d9c8db89b402cb6d0; no voters: 5e711b730f1a4307a4049a64e51eecd4, 63d7e494c1c449ccbdf58c42eb024ea1
20:04:33.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:33.421259 28862 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
20:04:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:33.500170 28847 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 28818
20:04:34.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:34.073634 28818 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.972s user 0.000s sys 0.002s
20:04:34.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:34.078217 28818 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.977s user 0.000s sys 0.002s
20:04:34.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:32.103127 28849 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:34.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.093495 28818 server_base.cc:1061] running on GCE node
20:04:34.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:34.094900 28874 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:34.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.096519 28818 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:34.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.107793 28818 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.115990 28818 hybrid_clock.cc:648] HybridClock initialized: now 1781467474115940 us; error 52 us; skew 500 ppm
20:04:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.117180 28818 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:34 dist-test-slave-h0zs krb5kdc[27924](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467474, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:34.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.124560 28818 init.cc:377] Logged in from keytab as kudu/127.27.66.60@KRBTEST.COM (short username kudu)
20:04:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.132676 28818 webserver.cc:533] Webserver started at http://127.27.66.60:37759/ using document root <none> and password file <none>
20:04:34.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.133786 28818 fs_manager.cc:362] Metadata directory not provided
20:04:34.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.134078 28818 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.145859 28818 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.002s sys 0.005s
20:04:34.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.151501 28880 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:34.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.152705 28818 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.004s
20:04:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.153072 28818 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/data,/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/wal
20:04:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63d7e494c1c449ccbdf58c42eb024ea1"
20:04:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:14 on dist-test-slave-h0zs"
20:04:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.155128 28818 fs_report.cc:389] FS layout report
20:04:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/wal
20:04:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/wal
20:04:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/data/data
20:04:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:34.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:34.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:34.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:34.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:34.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:34.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:34.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:34.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.228650 28818 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:34.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.236498 28818 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:34.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.237083 28818 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.392166 28818 rpc_server.cc:307] RPC server started. Bound to: 127.27.66.60:45873
20:04:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.392239 28931 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.66.60:45873 every 8 connection(s)
20:04:34.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.395489 28818 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-2/data/info.pb
20:04:34.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.402808 27912 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu as pid 28818
20:04:34.404 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:34.405 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.27.66.61:34759
20:04:34.405 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:04:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.407390 27912 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/wal
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/data
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/logs
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/data/info.pb
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.66.61:34759
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.66.61
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33003
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab
20:04:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.66.61
20:04:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.66.20:44487
20:04:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:34.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.66.62:40091,127.27.66.61:34759,127.27.66.60:45873
20:04:34.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:34.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41857/jwks.json
20:04:34.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:04:34.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:34.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.421522 28932 sys_catalog.cc:263] Verifying existing consensus state
20:04:34.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.436429 28932 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1: Bootstrap starting.
20:04:34.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.495257 28932 log.cc:826] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1: Log is configured to *not* fsync() on all Append() calls
20:04:34.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.522672 28932 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:34.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.523696 28932 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1: Bootstrap complete.
20:04:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.561399 28932 raft_consensus.cc:359] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.565086 28932 raft_consensus.cc:740] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 63d7e494c1c449ccbdf58c42eb024ea1, State: Initialized, Role: FOLLOWER
20:04:34.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.566452 28932 consensus_queue.cc:260] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:34.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.578259 28932 sys_catalog.cc:565] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [sys.catalog]: configured and running, proceeding with master startup.
20:04:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.581154 28936 sys_catalog.cc:455] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.582017 28936 sys_catalog.cc:458] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [sys.catalog]: This master's current role is: FOLLOWER
20:04:34.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.671579 28947 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:04:34.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:34 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467472, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.2@KRBTEST.COM for kudu/127.27.66.60@KRBTEST.COM
20:04:34.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:34 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467474, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.60@KRBTEST.COM for kudu/127.27.66.62@KRBTEST.COM
20:04:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.802953 28552 heartbeater.cc:344] Connected to a master server at 127.27.66.60:45873
20:04:34.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.844099 28702 heartbeater.cc:344] Connected to a master server at 127.27.66.60:45873
20:04:34.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:34 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467468, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.1@KRBTEST.COM for kudu/127.27.66.60@KRBTEST.COM
20:04:34.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.784574 28818 master_runner.cc:187] Error getting master registration for 127.27.66.61:34759: Network error: Client connection negotiation failed: client connection to 127.27.66.61:34759: connect: Connection refused (error 111),
20:04:34.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.863536 28818 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:04:34.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.866360 28896 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "33d0aab0850a4967a59f650971b7ae40" instance_seqno: 1781467467344730) as {username='kudu', principal='kudu/127.27.66.3@KRBTEST.COM'} at 127.27.66.3:46163; Asking this server to re-register.
20:04:34.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.868098 28702 heartbeater.cc:461] Registering TS with master...
20:04:34.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.867964 28897 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c6724f14c8514542a61de931d0e8cb91" instance_seqno: 1781467462323475) as {username='kudu', principal='kudu/127.27.66.2@KRBTEST.COM'} at 127.27.66.2:50951; Asking this server to re-register.
20:04:34.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.869403 28552 heartbeater.cc:461] Registering TS with master...
20:04:34.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.871588 28897 ts_manager.cc:194] Registered new tserver with Master: 33d0aab0850a4967a59f650971b7ae40 (127.27.66.3:38985)
20:04:34.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.874097 28896 ts_manager.cc:194] Registered new tserver with Master: c6724f14c8514542a61de931d0e8cb91 (127.27.66.2:44429)
20:04:34.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.890789 28400 heartbeater.cc:344] Connected to a master server at 127.27.66.60:45873
20:04:34.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.893776 28897 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "46c5297fad4b4e75b309237170619903" instance_seqno: 1781467459404385) as {username='kudu', principal='kudu/127.27.66.1@KRBTEST.COM'} at 127.27.66.1:43095; Asking this server to re-register.
20:04:34.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.897889 28400 heartbeater.cc:461] Registering TS with master...
20:04:34.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:34.901230 28897 ts_manager.cc:194] Registered new tserver with Master: 46c5297fad4b4e75b309237170619903 (127.27.66.1:38573)
20:04:35.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:35.057861 28934 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:35.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:35.058728 28934 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:35.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:35.059839 28934 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:35.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:35.124158 28934 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:04:35.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:35.124485 28934 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:35.124943 28934 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:35.125401 28934 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:35.125782 28934 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:35.126139 28934 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:35.126412 28934 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41857/jwks.json
20:04:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.197275 28934 master_runner.cc:387] Master server non-default flags:
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.66.20:44487
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/data
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/wal
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.66.62:40091,127.27.66.61:34759,127.27.66.60:45873
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/krb5kdc/kudu.keytab
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.66.61
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.66.61:34759
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41857/jwks.json
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/data/info.pb
20:04:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.66.61
20:04:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33003
20:04:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/logs
20:04:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6b1bbe17b8ab938291541389a90c67c358bba716
20:04:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jun 2026 19:43:21 UTC on e7f111948823
20:04:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12395
20:04:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.198946 28934 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.201140 28934 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:35.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:35.226892 28965 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:35.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.414561 28971 raft_consensus.cc:493] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:04:35.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.417352 28971 raft_consensus.cc:515] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:35.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.446907 28971 leader_election.cc:290] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5e711b730f1a4307a4049a64e51eecd4 (127.27.66.61:34759), 63d7e494c1c449ccbdf58c42eb024ea1 (127.27.66.60:45873)
20:04:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:35.490764 28767 leader_election.cc:336] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 5e711b730f1a4307a4049a64e51eecd4 (127.27.66.61:34759): Network error: Client connection negotiation failed: client connection to 127.27.66.61:34759: connect: Connection refused (error 111)
20:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:35 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467471, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.62@KRBTEST.COM for kudu/127.27.66.60@KRBTEST.COM
20:04:35.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.538807 28907 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "23a268f069b6459d9c8db89b402cb6d0" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" is_pre_election: true
20:04:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.539997 28907 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 23a268f069b6459d9c8db89b402cb6d0 in term 1.
20:04:35.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.541906 28765 leader_election.cc:304] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 23a268f069b6459d9c8db89b402cb6d0, 63d7e494c1c449ccbdf58c42eb024ea1; no voters: 5e711b730f1a4307a4049a64e51eecd4
20:04:35.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.542968 28971 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 1 FOLLOWER]: Leader pre-election won for term 2
20:04:35.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.543406 28971 raft_consensus.cc:493] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:04:35.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.543788 28971 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 1 FOLLOWER]: Advancing to term 2
20:04:35.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.575029 28971 raft_consensus.cc:515] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:35.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.578828 28907 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "23a268f069b6459d9c8db89b402cb6d0" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "63d7e494c1c449ccbdf58c42eb024ea1"
20:04:35.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.579391 28907 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [term 1 FOLLOWER]: Advancing to term 2
20:04:35.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:35.583565 28767 leader_election.cc:336] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 5e711b730f1a4307a4049a64e51eecd4 (127.27.66.61:34759): Network error: Client connection negotiation failed: client connection to 127.27.66.61:34759: connect: Connection refused (error 111)
20:04:35.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.592864 28971 leader_election.cc:290] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [CANDIDATE]: Term 2 election: Requested vote from peers 5e711b730f1a4307a4049a64e51eecd4 (127.27.66.61:34759), 63d7e494c1c449ccbdf58c42eb024ea1 (127.27.66.60:45873)
20:04:35.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.592278 28907 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 23a268f069b6459d9c8db89b402cb6d0 in term 2.
20:04:35.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.606294 28765 leader_election.cc:304] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 23a268f069b6459d9c8db89b402cb6d0, 63d7e494c1c449ccbdf58c42eb024ea1; no voters: 5e711b730f1a4307a4049a64e51eecd4
20:04:35.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.607707 28971 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 2 FOLLOWER]: Leader election won for term 2
20:04:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.615392 28971 raft_consensus.cc:697] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [term 2 LEADER]: Becoming Leader. State: Replica: 23a268f069b6459d9c8db89b402cb6d0, State: Running, Role: LEADER
20:04:35.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.617096 28971 consensus_queue.cc:237] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.640035 28976 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 23a268f069b6459d9c8db89b402cb6d0. Latest consensus state: current_term: 2 leader_uuid: "23a268f069b6459d9c8db89b402cb6d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.641075 28976 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [sys.catalog]: This master's current role is: LEADER
20:04:35.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.658941 28978 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
20:04:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.672283 28978 catalog_manager.cc:1519] Initializing Kudu cluster ID...
20:04:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.678846 28978 catalog_manager.cc:1294] Loaded cluster ID: c7fa964344734d38bee73a3a28382aaa
20:04:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.679528 28978 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
20:04:35.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.686689 28978 catalog_manager.cc:1539] Loading token signing keys...
20:04:35.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.693437 28978 catalog_manager.cc:6080] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0: Loaded TSK: 0
20:04:35.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.695464 28978 catalog_manager.cc:1549] Initializing in-progress tserver states...
20:04:35.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.724342 28947 catalog_manager.cc:1294] Loaded cluster ID: c7fa964344734d38bee73a3a28382aaa
20:04:35.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.728451 28947 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1: loading cluster ID for follower catalog manager: success
20:04:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.743872 28947 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1: acquiring CA information for follower catalog manager: success
20:04:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:35.752858 28947 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:04:36.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:36.040046 28907 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [term 2 FOLLOWER]: Refusing update from remote peer 23a268f069b6459d9c8db89b402cb6d0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
20:04:36.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:36.042893 28976 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
20:04:36.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:36.064497 28767 consensus_peers.cc:597] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 -> Peer 5e711b730f1a4307a4049a64e51eecd4 (127.27.66.61:34759): Couldn't send request to peer 5e711b730f1a4307a4049a64e51eecd4. Status: Network error: Client connection negotiation failed: client connection to 127.27.66.61:34759: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:04:36.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:36.090903 28971 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "23a268f069b6459d9c8db89b402cb6d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:36.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:36.091840 28971 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23a268f069b6459d9c8db89b402cb6d0 [sys.catalog]: This master's current role is: LEADER
20:04:36.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:36.113073 28984 sys_catalog.cc:455] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 23a268f069b6459d9c8db89b402cb6d0. Latest consensus state: current_term: 2 leader_uuid: "23a268f069b6459d9c8db89b402cb6d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:36.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:36.115226 28984 sys_catalog.cc:458] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [sys.catalog]: This master's current role is: FOLLOWER
20:04:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:36.125684 28988 sys_catalog.cc:455] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "23a268f069b6459d9c8db89b402cb6d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:36.126533 28988 sys_catalog.cc:458] T 00000000000000000000000000000000 P 63d7e494c1c449ccbdf58c42eb024ea1 [sys.catalog]: This master's current role is: FOLLOWER
20:04:36.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:36.630167 28717 master_runner.cc:187] Error getting master registration for 127.27.66.61:34759: Network error: Client connection negotiation failed: client connection to 127.27.66.61:34759: connect: Connection refused (error 111),
20:04:36.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:36.630564 28717 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:04:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:36.762070 28706 heartbeater.cc:499] Master 127.27.66.62:40091 was elected leader, sending a full tablet report...
20:04:36.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:36.810472 28405 heartbeater.cc:499] Master 127.27.66.62:40091 was elected leader, sending a full tablet report...
20:04:36.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:36 dist-test-slave-h0zs krb5kdc[27924](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467476, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:36.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:36.992353 28585 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:37.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:36.627465 28964 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 28934
20:04:37.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:37.225910 28964 kernel_stack_watchdog.cc:198] Thread 28934 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 394ms:
20:04:37.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:04:37.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:04:37.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:04:37.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:04:37.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:35.230893 28966 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:37.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:37.227110 28934 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.995s user 0.000s sys 0.002s
20:04:37.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:37.227522 28934 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.996s user 0.000s sys 0.002s
20:04:37.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:37.229835 28969 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:37.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:37.233314 28967 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1997 milliseconds
20:04:37.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.234095 28934 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.235347 28934 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:37.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.238734 28934 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:37.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.240198 28934 hybrid_clock.cc:648] HybridClock initialized: now 1781467477240161 us; error 44 us; skew 500 ppm
20:04:37.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.241159 28934 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:37.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:37 dist-test-slave-h0zs krb5kdc[27924](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467477, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:37.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.247722 28934 init.cc:377] Logged in from keytab as kudu/127.27.66.61@KRBTEST.COM (short username kudu)
20:04:37.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.253909 28934 webserver.cc:533] Webserver started at http://127.27.66.61:33003/ using document root <none> and password file <none>
20:04:37.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.255045 28934 fs_manager.cc:362] Metadata directory not provided
20:04:37.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.255357 28934 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.264649 28934 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.000s
20:04:37.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.270649 29002 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:37.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.271838 28934 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
20:04:37.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.272207 28934 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/data,/tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/wal
20:04:37.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e711b730f1a4307a4049a64e51eecd4"
20:04:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-14 20:04:11 on dist-test-slave-h0zs"
20:04:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.274210 28934 fs_report.cc:389] FS layout report
20:04:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/wal
20:04:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/wal
20:04:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/data/data
20:04:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:37.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.335906 28934 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:37.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.346715 28934 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:37.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.347275 28934 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.456028 28934 rpc_server.cc:307] RPC server started. Bound to: 127.27.66.61:34759
20:04:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.456079 29053 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.66.61:34759 every 8 connection(s)
20:04:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.460170 28934 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskCk90tC/test-tmp/mini-kudu-cluster3155043012870535849/master-1/data/info.pb
20:04:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.461853 27912 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu as pid 28934
20:04:37.462 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:37.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.479650 29054 sys_catalog.cc:263] Verifying existing consensus state
20:04:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.487542 29054 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4: Bootstrap starting.
20:04:37.519 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.66.62:40091(127.27.66.62:40091)] connecting to peer
20:04:37.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.541085 29054 log.cc:826] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4: Log is configured to *not* fsync() on all Append() calls
20:04:37.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.573179 29054 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:37.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.574291 29054 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4: Bootstrap complete.
20:04:37.588 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.66.60:45873(127.27.66.60:45873)] connecting to peer
20:04:37.601 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.66.61:34759(127.27.66.61:34759)] connecting to peer
20:04:37.619 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.27.66.60:45873(127.27.66.60:45873)] Successfully connected to peer
20:04:37.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.608781 29054 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:37.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.623858 29054 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5e711b730f1a4307a4049a64e51eecd4, State: Initialized, Role: FOLLOWER
20:04:37.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.624938 29054 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } }
20:04:37.629 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.27.66.61:34759(127.27.66.61:34759)] Successfully connected to peer
20:04:37.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.644745 29063 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:37.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.645771 29063 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [sys.catalog]: This master's current role is: FOLLOWER
20:04:37.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.646461 29054 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [sys.catalog]: configured and running, proceeding with master startup.
20:04:37.680 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.27.66.62:40091(127.27.66.62:40091)] Successfully connected to peer
20:04:37.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.693464 28554 heartbeater.cc:499] Master 127.27.66.62:40091 was elected leader, sending a full tablet report...
20:04:37.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:37 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467471, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.62@KRBTEST.COM for kudu/127.27.66.61@KRBTEST.COM
20:04:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.721783 29029 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [term 1 FOLLOWER]: Advancing to term 2
20:04:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.753427 29076 catalog_manager.cc:1294] Loaded cluster ID: c7fa964344734d38bee73a3a28382aaa
20:04:37.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.753926 29076 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4: loading cluster ID for follower catalog manager: success
20:04:37.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.786202 29076 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4: acquiring CA information for follower catalog manager: success
20:04:37.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 14 20:04:37 dist-test-slave-h0zs krb5kdc[27924](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781467477, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.66.61@KRBTEST.COM for kudu/127.27.66.62@KRBTEST.COM
20:04:37.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.795614 29063 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 23a268f069b6459d9c8db89b402cb6d0. Latest consensus state: current_term: 2 leader_uuid: "23a268f069b6459d9c8db89b402cb6d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:37.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.799126 29063 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [sys.catalog]: This master's current role is: FOLLOWER
20:04:37.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.798324 29083 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "23a268f069b6459d9c8db89b402cb6d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23a268f069b6459d9c8db89b402cb6d0" member_type: VOTER last_known_addr { host: "127.27.66.62" port: 40091 } } peers { permanent_uuid: "5e711b730f1a4307a4049a64e51eecd4" member_type: VOTER last_known_addr { host: "127.27.66.61" port: 34759 } } peers { permanent_uuid: "63d7e494c1c449ccbdf58c42eb024ea1" member_type: VOTER last_known_addr { host: "127.27.66.60" port: 45873 } } }
20:04:37.802 [WARN - kudu-nio-0] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
20:04:37.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.802687 29083 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4 [sys.catalog]: This master's current role is: FOLLOWER
20:04:37.806 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.66.60:45873(127.27.66.60:45873)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
20:04:37.807 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.66.60:45873: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
20:04:37.808 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.27.66.61:34759(127.27.66.61:34759)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
20:04:37.808 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.27.66.61:34759: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
20:04:37.811 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.27.66.61:34759(127.27.66.61:34759)] handling channelInactive
20:04:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:37.810770 29062 negotiation.cc:336] Failed RPC negotiation. Trace:
20:04:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.615762 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:39012 (local address 127.27.66.60:45873)
20:04:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.617304 (+ 1542us) server_negotiation.cc:207] Beginning negotiation
20:04:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.617332 (+ 28us) server_negotiation.cc:400] Waiting for connection header
20:04:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.753369 (+136037us) server_negotiation.cc:408] Connection header received
20:04:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.797026 (+ 43657us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:04:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.797058 (+ 32us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:04:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.797309 (+ 251us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:04:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.797493 (+ 184us) server_negotiation.cc:227] Negotiated authn=SASL
20:04:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.808673 (+ 11180us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:39012: BlockingRecv error: recv got EOF from 127.0.0.1:39012 (error 108)
20:04:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":912,"thread_start_us":619,"threads_started":1}
20:04:37.814 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.27.66.60:45873(127.27.66.60:45873)] handling channelInactive
20:04:37.815 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.66.62:40091(127.27.66.62:40091)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
20:04:37.815 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.66.62:40091: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
20:04:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:37.813741 29064 negotiation.cc:336] Failed RPC negotiation. Trace:
20:04:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.626355 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:41816 (local address 127.27.66.61:34759)
20:04:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.643399 (+ 17044us) server_negotiation.cc:207] Beginning negotiation
20:04:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.643448 (+ 49us) server_negotiation.cc:400] Waiting for connection header
20:04:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.744041 (+100593us) server_negotiation.cc:408] Connection header received
20:04:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.783949 (+ 39908us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:04:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.783982 (+ 33us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:04:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.784190 (+ 208us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:04:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.784363 (+ 173us) server_negotiation.cc:227] Negotiated authn=SASL
20:04:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.811883 (+ 27520us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:41816: BlockingRecv error: recv got EOF from 127.0.0.1:41816 (error 108)
20:04:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":16255,"thread_start_us":16893,"threads_started":1}
20:04:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.816133 29076 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 5e711b730f1a4307a4049a64e51eecd4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:04:37.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.821465 28553 heartbeater.cc:344] Connected to a master server at 127.27.66.61:34759
20:04:37.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.827730 29019 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c6724f14c8514542a61de931d0e8cb91" instance_seqno: 1781467462323475) as {username='kudu', principal='kudu/127.27.66.2@KRBTEST.COM'} at 127.27.66.2:60297; Asking this server to re-register.
20:04:37.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.834228 28553 heartbeater.cc:461] Registering TS with master...
20:04:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.837582 29019 ts_manager.cc:194] Registered new tserver with Master: c6724f14c8514542a61de931d0e8cb91 (127.27.66.2:44429)
20:04:37.839 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=368), Traces: [0ms] refreshing cache from master, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.66.62:40091, [116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.66.60:45873, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.66.61:34759, [335ms] Sub RPC ConnectToMaster: received response from server master-127.27.66.60:45873: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [336ms] Sub RPC ConnectToMaster: received response from server master-127.27.66.61:34759: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [343ms] Sub RPC ConnectToMaster: received response from server master-127.27.66.62:40091: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.27.66.62:40091,127.27.66.60:45873,127.27.66.61:34759). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.27.66.62:40091,127.27.66.60:45873,127.27.66.61:34759). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
20:04:37.843 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=375), Traces: [0ms] refreshing cache from master, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.66.62:40091, [116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.66.60:45873, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.66.61:34759, [335ms] Sub RPC ConnectToMaster: received response from server master-127.27.66.60:45873: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [336ms] Sub RPC ConnectToMaster: received response from server master-127.27.66.61:34759: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [343ms] Sub RPC ConnectToMaster: received response from server master-127.27.66.62:40091: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) with no deferred attached!
20:04:37.846 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.27.66.62:40091(127.27.66.62:40091)] disconnecting while in state TERMINATED
20:04:37.847 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.27.66.60:45873(127.27.66.60:45873)] disconnecting while in state TERMINATED
20:04:37.848 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.27.66.61:34759(127.27.66.61:34759)] disconnecting while in state TERMINATED
20:04:37.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:37.853041 29061 negotiation.cc:336] Failed RPC negotiation. Trace:
20:04:37.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.613789 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:40992 (local address 127.27.66.62:40091)
20:04:37.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.646576 (+ 32787us) server_negotiation.cc:207] Beginning negotiation
20:04:37.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.646603 (+ 27us) server_negotiation.cc:400] Waiting for connection header
20:04:37.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.744490 (+ 97887us) server_negotiation.cc:408] Connection header received
20:04:37.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.782130 (+ 37640us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:04:37.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.782160 (+ 30us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:04:37.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.782400 (+ 240us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:04:37.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.782575 (+ 175us) server_negotiation.cc:227] Negotiated authn=SASL
20:04:37.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.851662 (+ 69087us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:40992: BlockingRecv error: recv got EOF from 127.0.0.1:40992 (error 108)
20:04:37.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":32330,"thread_start_us":32795,"threads_started":1}
20:04:37.855 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:04:37.866 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:04:37.867 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.27.66.62:40091(127.27.66.62:40091)] handling channelInactive
20:04:37.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.933624 27912 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu with pid 28238
20:04:37.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.951884 28703 heartbeater.cc:344] Connected to a master server at 127.27.66.61:34759
20:04:37.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.954718 29019 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "33d0aab0850a4967a59f650971b7ae40" instance_seqno: 1781467467344730) as {username='kudu', principal='kudu/127.27.66.3@KRBTEST.COM'} at 127.27.66.3:40283; Asking this server to re-register.
20:04:37.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.956233 28703 heartbeater.cc:461] Registering TS with master...
20:04:37.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.959147 29019 ts_manager.cc:194] Registered new tserver with Master: 33d0aab0850a4967a59f650971b7ae40 (127.27.66.3:38985)
20:04:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260614 20:04:37.971297 29064 negotiation.cc:336] Failed RPC negotiation. Trace:
20:04:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.917424 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.27.66.1:39245 (local address 127.27.66.61:34759)
20:04:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.918107 (+ 683us) server_negotiation.cc:207] Beginning negotiation
20:04:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.918134 (+ 27us) server_negotiation.cc:400] Waiting for connection header
20:04:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.929940 (+ 11806us) server_negotiation.cc:408] Connection header received
20:04:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.930333 (+ 393us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:04:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.930373 (+ 40us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:04:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.930647 (+ 274us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:04:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.930812 (+ 165us) server_negotiation.cc:227] Negotiated authn=CERTIFICATE
20:04:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.933413 (+ 2601us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
20:04:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.935950 (+ 2537us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
20:04:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0614 20:04:37.970121 (+ 34171us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.27.66.1:39245: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:04:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":128}
20:04:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:37.971428 27912 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu with pid 28414
20:04:38.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:38.008303 27912 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu with pid 28567
20:04:38.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:38.036773 27912 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu with pid 28717
20:04:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:38.086207 27912 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu with pid 28934
20:04:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:38.113374 27912 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu with pid 28818
20:04:39.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-14T20:04:39Z chronyd exiting
20:04:40.391 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:04:40.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:40.394052 27188 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu with pid 27487
20:04:40.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:40.423568 27188 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu with pid 27628
20:04:40.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:40.454161 27188 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu with pid 27770
20:04:40.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:40.485962 27188 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu with pid 27200
20:04:40.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:40.519990 27188 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu with pid 27270
20:04:40.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260614 20:04:40.553953 27188 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskCk90tC/build/tsan/bin/kudu with pid 27345
20:04:41.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-14T20:04:41Z chronyd exiting
20:04:42.804 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.27.66.62:40091,127.27.66.60:45873,127.27.66.61:34759). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.27.66.62:40091,127.27.66.60:45873,127.27.66.61:34759). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more