20:00:22.185 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:00:22.199 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskwpaMk5/test-tmp
20:00:22.206 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:00:22.209 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskwpaMk5/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:00:22.394 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:00:22.588 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:00:22.589 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:00:22.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:22.595036 23562 mini_oidc.cc:150] Starting JWKS server
20:00:22.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:22.599260 23562 webserver.cc:533] Webserver started at http://127.0.0.1:42971/ using document root <none> and password file <none>
20:00:22.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:22.599934 23562 mini_oidc.cc:186] Starting OIDC Discovery server
20:00:22.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:22.602029 23562 webserver.cc:533] Webserver started at http://127.0.0.1:36969/ using document root <none> and password file <none>
20:00:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T20:00:22Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:00:22.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T20:00:22Z Disabled control of system clock
20:00:22.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:22.667657 23562 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:22.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:22.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-0/wal
20:00:22.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-0/data
20:00:22.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:22.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:22.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:22.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:22.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:22.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:22.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-0/logs
20:00:22.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-0/data/info.pb
20:00:22.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:22.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:22.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:22.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:22.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:22.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:22.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:22.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:22.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:00:22.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:22.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:22.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:22.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.190:43005
20:00:22.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.190
20:00:22.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:22.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:41767
20:00:22.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:22.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.2.190:43005,127.23.2.189:44311,127.23.2.188:40447
20:00:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42971/jwks.json with env {}
20:00:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:22.972363 23577 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:00:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:22.972959 23577 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:22.973624 23577 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:00:23.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:23.006043 23577 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:00:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:23.006321 23577 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:00:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:23.006556 23577 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:00:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:23.006851 23577 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:00:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:23.007195 23577 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:00:23.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:23.007458 23577 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42971/jwks.json
20:00:23.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:23.043738 23577 master_runner.cc:387] Master server non-default flags:
20:00:23.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-0/logs
20:00:23.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:23.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:23.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:23.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:41767
20:00:23.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:23.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:23.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-0/data
20:00:23.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-0/wal
20:00:23.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:23.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.2.190:43005,127.23.2.189:44311,127.23.2.188:40447
20:00:23.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:23.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:23.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:23.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.190:43005
20:00:23.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:23.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:23.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:23.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42971/jwks.json
20:00:23.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:23.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:23.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-0/data/info.pb
20:00:23.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.190
20:00:23.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:23.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:23.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:23.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:23.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:23.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:23.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:23.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:00:23.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:00:23.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 565fa2a0c8b8655dfe08c1dca95e4b4d9408fc9c
20:00:23.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:23.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 19:45:48 UTC on e7f111948823
20:00:23.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13311
20:00:23.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:23.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:23.045272 23577 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:23.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:23.047268 23577 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:00:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:23.061053 23584 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:24.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:24.464188 23583 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee60 after lost signal to thread 23577
20:00:24.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:24.872607 23583 kernel_stack_watchdog.cc:198] Thread 23577 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
20:00:24.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:00:24.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:00:24.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:00:24.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:00:24.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:24.874307 23577 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.814s user 0.658s sys 1.156s
20:00:24.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:24.874712 23577 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.814s user 0.658s sys 1.156s
20:00:24.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:23.061316 23585 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:24.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:24.875648 23586 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1812 milliseconds
20:00:24.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:24.876511 23587 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:24.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.876981 23577 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:00:24.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.880308 23577 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:24.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.883111 23577 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:24.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.884497 23577 hybrid_clock.cc:648] HybridClock initialized: now 1786564824884400 us; error 109 us; skew 500 ppm
20:00:24.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.885478 23577 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:24.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.891634 23577 webserver.cc:533] Webserver started at http://127.23.2.190:35881/ using document root <none> and password file <none>
20:00:24.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.892762 23577 fs_manager.cc:362] Metadata directory not provided
20:00:24.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.893014 23577 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:24.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.893646 23577 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:00:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.898128 23577 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-0/data/instance:
20:00:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "32b9a40abcb0416ea5643975d54f6ee0"
20:00:24.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:24 on dist-test-slave-1jjb"
20:00:24.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.899552 23577 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-0/wal/instance:
20:00:24.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "32b9a40abcb0416ea5643975d54f6ee0"
20:00:24.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:24 on dist-test-slave-1jjb"
20:00:24.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.907445 23577 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
20:00:24.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.913437 23594 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:24.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.914876 23577 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
20:00:24.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.915241 23577 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-0/data,/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-0/wal
20:00:24.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "32b9a40abcb0416ea5643975d54f6ee0"
20:00:24.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:24 on dist-test-slave-1jjb"
20:00:24.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.915695 23577 fs_report.cc:389] FS layout report
20:00:24.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:24.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-0/wal
20:00:24.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-0/wal
20:00:24.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-0/data/data
20:00:24.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:24.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:24.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:24.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:24.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:24.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:24.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:24.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:24.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:24.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:24.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:24.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.979290 23577 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:00:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.981040 23577 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:24.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:24.981607 23577 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:25.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:25.055269 23577 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.190:43005
20:00:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:25.055353 23654 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.2.190:43005 every 8 connection(s)
20:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:25.058414 23577 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-0/data/info.pb
20:00:25.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:25.059283 23562 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu as pid 23577
20:00:25.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:25.059700 23562 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-0/wal/instance
20:00:25.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:25.064283 23562 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:25.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:25.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-1/wal
20:00:25.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-1/data
20:00:25.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:25.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:25.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:25.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:25.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-1/logs
20:00:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-1/data/info.pb
20:00:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:25.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:25.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:25.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:00:25.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:25.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:25.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:25.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.189:44311
20:00:25.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.189
20:00:25.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:25.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:41767
20:00:25.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:25.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:25.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:25.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:25.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.2.190:43005,127.23.2.189:44311,127.23.2.188:40447
20:00:25.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:25.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42971/jwks.json with env {}
20:00:25.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:25.066111 23655 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:00:25.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:25.083638 23655 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } has no permanent_uuid. Determining permanent_uuid...
20:00:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:25.112157 23655 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } has no permanent_uuid. Determining permanent_uuid...
20:00:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:25.114972 23598 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.2.189:44311: connect: Connection refused (error 111)
20:00:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:25.119864 23655 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.2.189:44311: Network error: Client connection negotiation failed: client connection to 127.23.2.189:44311: connect: Connection refused (error 111)
20:00:25.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:25.169421 23655 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } attempt: 1
20:00:25.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:25.174646 23655 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.2.189:44311: Network error: Client connection negotiation failed: client connection to 127.23.2.189:44311: connect: Connection refused (error 111)
20:00:25.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:25.243160 23655 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } attempt: 2
20:00:25.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:25.248023 23655 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.2.189:44311: Network error: Client connection negotiation failed: client connection to 127.23.2.189:44311: connect: Connection refused (error 111)
20:00:25.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:25.339450 23655 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } attempt: 3
20:00:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:25.343976 23655 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.2.189:44311: Network error: Client connection negotiation failed: client connection to 127.23.2.189:44311: connect: Connection refused (error 111)
20:00:25.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:25.401461 23657 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:00:25.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:25.402230 23657 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:25.403103 23657 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:00:25.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:25.437266 23657 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:00:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:25.437667 23657 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:00:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:25.438022 23657 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:00:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:25.438370 23657 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:00:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:25.438748 23657 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:00:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:25.439051 23657 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42971/jwks.json
20:00:25.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:25.480557 23657 master_runner.cc:387] Master server non-default flags:
20:00:25.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-1/logs
20:00:25.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:25.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:25.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:25.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:41767
20:00:25.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:25.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:25.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-1/data
20:00:25.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-1/wal
20:00:25.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:25.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.2.190:43005,127.23.2.189:44311,127.23.2.188:40447
20:00:25.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:25.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:25.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:25.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.189:44311
20:00:25.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:25.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42971/jwks.json
20:00:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-1/data/info.pb
20:00:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.189
20:00:25.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:25.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:25.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:25.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:25.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:25.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:25.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:25.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:00:25.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:00:25.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 565fa2a0c8b8655dfe08c1dca95e4b4d9408fc9c
20:00:25.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:25.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 19:45:48 UTC on e7f111948823
20:00:25.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13311
20:00:25.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:25.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:25.482313 23657 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:25.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:25.484764 23657 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:00:25.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:25.487387 23655 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } attempt: 4
20:00:25.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:25.492640 23655 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.2.189:44311: Network error: Client connection negotiation failed: client connection to 127.23.2.189:44311: connect: Connection refused (error 111)
20:00:25.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:25.499609 23667 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:25.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:25.792037 23655 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } attempt: 5
20:00:25.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:25.804445 23655 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.2.189:44311: Network error: Client connection negotiation failed: client connection to 127.23.2.189:44311: connect: Connection refused (error 111)
20:00:26.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:26.352000 23655 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } attempt: 6
20:00:26.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:26.384924 23655 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.2.189:44311: Network error: Client connection negotiation failed: client connection to 127.23.2.189:44311: connect: Connection refused (error 111)
20:00:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:25.500888 23669 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:27.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:26.903640 23666 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee60 after lost signal to thread 23657
20:00:27.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.320379 23666 kernel_stack_watchdog.cc:198] Thread 23657 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
20:00:27.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:00:27.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:00:27.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:00:27.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:00:27.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.325883 23657 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.824s user 0.004s sys 0.000s
20:00:27.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.326311 23657 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.824s user 0.004s sys 0.000s
20:00:27.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.328214 23673 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:27.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.328554 23670 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1826 milliseconds
20:00:27.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.328881 23657 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:00:27.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.330333 23657 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.340319 23657 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.342875 23657 hybrid_clock.cc:648] HybridClock initialized: now 1786564827342783 us; error 55 us; skew 500 ppm
20:00:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.344202 23657 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.353375 23657 webserver.cc:533] Webserver started at http://127.23.2.189:44673/ using document root <none> and password file <none>
20:00:27.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.354336 23657 fs_manager.cc:362] Metadata directory not provided
20:00:27.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.354583 23657 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:27.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.355006 23657 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:00:27.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.359764 23657 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-1/data/instance:
20:00:27.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63d2980890f64afaa34f41df2757293a"
20:00:27.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:27 on dist-test-slave-1jjb"
20:00:27.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.361240 23657 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-1/wal/instance:
20:00:27.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63d2980890f64afaa34f41df2757293a"
20:00:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:27 on dist-test-slave-1jjb"
20:00:27.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.369755 23657 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
20:00:27.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.376276 23681 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:27.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.377573 23657 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:00:27.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.378044 23657 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-1/data,/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-1/wal
20:00:27.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63d2980890f64afaa34f41df2757293a"
20:00:27.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:27 on dist-test-slave-1jjb"
20:00:27.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.378511 23657 fs_report.cc:389] FS layout report
20:00:27.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:27.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-1/wal
20:00:27.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-1/wal
20:00:27.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-1/data/data
20:00:27.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:27.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:27.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:27.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:27.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:27.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:27.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:27.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:27.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:27.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:27.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.434476 23657 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:00:27.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.436400 23657 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:27.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.436959 23657 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:27.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.445479 23655 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } attempt: 7
20:00:27.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.453471 23655 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.2.189:44311: Network error: Client connection negotiation failed: client connection to 127.23.2.189:44311: connect: Connection refused (error 111)
20:00:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.526365 23657 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.189:44311
20:00:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.526456 23741 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.2.189:44311 every 8 connection(s)
20:00:27.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.529451 23657 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-1/data/info.pb
20:00:27.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.530007 23562 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu as pid 23657
20:00:27.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.530354 23562 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-1/wal/instance
20:00:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.533432 23562 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:27.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:27.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-2/wal
20:00:27.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-2/data
20:00:27.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:27.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:27.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:27.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:27.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:27.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:27.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-2/logs
20:00:27.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-2/data/info.pb
20:00:27.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:27.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:27.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:27.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:27.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:27.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:27.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:27.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:27.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:00:27.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:27.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:27.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:27.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.188:40447
20:00:27.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.188
20:00:27.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:27.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:41767
20:00:27.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:27.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:27.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:27.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:27.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.2.190:43005,127.23.2.189:44311,127.23.2.188:40447
20:00:27.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:27.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42971/jwks.json with env {}
20:00:27.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.537569 23742 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:00:27.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.548786 23742 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } has no permanent_uuid. Determining permanent_uuid...
20:00:27.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.570235 23742 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } has no permanent_uuid. Determining permanent_uuid...
20:00:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.583341 23742 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } has no permanent_uuid. Determining permanent_uuid...
20:00:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.585795 23686 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.2.188:40447: connect: Connection refused (error 111)
20:00:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.590154 23742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.2.188:40447: Network error: Client connection negotiation failed: client connection to 127.23.2.188:40447: connect: Connection refused (error 111)
20:00:27.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.639739 23742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } attempt: 1
20:00:27.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.644936 23742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.2.188:40447: Network error: Client connection negotiation failed: client connection to 127.23.2.188:40447: connect: Connection refused (error 111)
20:00:27.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.713364 23742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } attempt: 2
20:00:27.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.718477 23742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.2.188:40447: Network error: Client connection negotiation failed: client connection to 127.23.2.188:40447: connect: Connection refused (error 111)
20:00:27.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.809934 23742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } attempt: 3
20:00:27.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.814407 23742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.2.188:40447: Network error: Client connection negotiation failed: client connection to 127.23.2.188:40447: connect: Connection refused (error 111)
20:00:27.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.854859 23744 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:00:27.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.855405 23744 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:27.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.855858 23744 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:00:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.888542 23744 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:00:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.888820 23744 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:00:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.889031 23744 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:00:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.889236 23744 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:00:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.889421 23744 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:00:27.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.889659 23744 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42971/jwks.json
20:00:27.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.927816 23744 master_runner.cc:387] Master server non-default flags:
20:00:27.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-2/logs
20:00:27.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:27.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:27.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:27.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:41767
20:00:27.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-2/data
20:00:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-2/wal
20:00:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.2.190:43005,127.23.2.189:44311,127.23.2.188:40447
20:00:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.188:40447
20:00:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42971/jwks.json
20:00:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-2/data/info.pb
20:00:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.188
20:00:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:00:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:00:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 565fa2a0c8b8655dfe08c1dca95e4b4d9408fc9c
20:00:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:27.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 19:45:48 UTC on e7f111948823
20:00:27.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13311
20:00:27.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:27.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.929454 23744 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:27.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.931675 23744 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:00:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.942239 23754 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:27.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:27.957847 23742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } attempt: 4
20:00:27.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.962514 23742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.2.188:40447: Network error: Client connection negotiation failed: client connection to 127.23.2.188:40447: connect: Connection refused (error 111)
20:00:28.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:28.261951 23742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } attempt: 5
20:00:28.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:28.269081 23742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.2.188:40447: Network error: Client connection negotiation failed: client connection to 127.23.2.188:40447: connect: Connection refused (error 111)
20:00:28.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:28.816653 23742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } attempt: 6
20:00:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:28.828140 23742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.2.188:40447: Network error: Client connection negotiation failed: client connection to 127.23.2.188:40447: connect: Connection refused (error 111)
20:00:29.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:27.944001 23755 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:29.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:29.178813 23756 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1233 milliseconds
20:00:29.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:29.179597 23757 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:29.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.179620 23744 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:00:29.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.182888 23744 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.186170 23744 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.187646 23744 hybrid_clock.cc:648] HybridClock initialized: now 1786564829187605 us; error 50 us; skew 500 ppm
20:00:29.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.188542 23744 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:29.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.195206 23744 webserver.cc:533] Webserver started at http://127.23.2.188:46625/ using document root <none> and password file <none>
20:00:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.196205 23744 fs_manager.cc:362] Metadata directory not provided
20:00:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.196426 23744 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.196887 23744 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:00:29.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.201345 23744 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-2/data/instance:
20:00:29.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2697fd68827a42f5b8d2a7c5782df5d0"
20:00:29.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:29 on dist-test-slave-1jjb"
20:00:29.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.202531 23744 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-2/wal/instance:
20:00:29.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2697fd68827a42f5b8d2a7c5782df5d0"
20:00:29.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:29 on dist-test-slave-1jjb"
20:00:29.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.210355 23744 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
20:00:29.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.216424 23767 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.217592 23744 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
20:00:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.217957 23744 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-2/data,/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-2/wal
20:00:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2697fd68827a42f5b8d2a7c5782df5d0"
20:00:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:29 on dist-test-slave-1jjb"
20:00:29.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.218416 23744 fs_report.cc:389] FS layout report
20:00:29.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:29.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-2/wal
20:00:29.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-2/wal
20:00:29.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-2/data/data
20:00:29.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:29.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:29.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:29.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:29.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:29.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:29.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:29.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:29.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:29.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:29.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:29.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.283864 23744 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:00:29.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.285616 23744 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:29.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.286154 23744 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:29.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.362140 23744 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.188:40447
20:00:29.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.362224 23824 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.2.188:40447 every 8 connection(s)
20:00:29.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.365412 23744 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-2/data/info.pb
20:00:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.367728 23562 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu as pid 23744
20:00:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.368347 23562 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/master-2/wal/instance
20:00:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.371899 23825 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:00:29.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.386909 23825 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } has no permanent_uuid. Determining permanent_uuid...
20:00:29.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.411051 23825 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } has no permanent_uuid. Determining permanent_uuid...
20:00:29.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.422384 23825 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } has no permanent_uuid. Determining permanent_uuid...
20:00:29.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.439230 23825 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0: Bootstrap starting.
20:00:29.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.446717 23825 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0: Neither blocks nor log segments found. Creating new log.
20:00:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.449066 23825 log.cc:826] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0: Log is configured to *not* fsync() on all Append() calls
20:00:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.453897 23825 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0: No bootstrap required, opened a new log
20:00:29.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.477289 23825 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } }
20:00:29.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.478241 23825 raft_consensus.cc:385] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:00:29.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.478577 23825 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2697fd68827a42f5b8d2a7c5782df5d0, State: Initialized, Role: FOLLOWER
20:00:29.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.479561 23825 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } }
20:00:29.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.483479 23835 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } } }
20:00:29.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.484686 23835 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0 [sys.catalog]: This master's current role is: FOLLOWER
20:00:29.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.485044 23825 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0 [sys.catalog]: configured and running, proceeding with master startup.
20:00:29.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:29.512120 23847 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:00:29.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:29.512454 23847 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:00:29.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.507800 23744 master_runner.cc:187] Error getting master registration for 127.23.2.190:43005: OK, instance_id { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" instance_seqno: 1786564825034262 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:00:29.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.530368 23744 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:00:29.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:29.541577 23774 tablet.cc:2406] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:00:29.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.543867 23655 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } attempt: 8
20:00:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.559834 23655 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } has no permanent_uuid. Determining permanent_uuid...
20:00:29.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.576874 23655 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0: Bootstrap starting.
20:00:29.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.582499 23655 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0: Neither blocks nor log segments found. Creating new log.
20:00:29.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.584466 23655 log.cc:826] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0: Log is configured to *not* fsync() on all Append() calls
20:00:29.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.589221 23655 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0: No bootstrap required, opened a new log
20:00:29.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.609241 23655 raft_consensus.cc:359] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } }
20:00:29.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.610031 23655 raft_consensus.cc:385] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:00:29.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.610308 23655 raft_consensus.cc:740] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 32b9a40abcb0416ea5643975d54f6ee0, State: Initialized, Role: FOLLOWER
20:00:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.611333 23655 consensus_queue.cc:260] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } }
20:00:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.614274 23851 sys_catalog.cc:455] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } } }
20:00:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.615417 23851 sys_catalog.cc:458] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [sys.catalog]: This master's current role is: FOLLOWER
20:00:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.616245 23655 sys_catalog.cc:565] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [sys.catalog]: configured and running, proceeding with master startup.
20:00:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:29.650527 23864 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:00:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:29.651320 23864 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:00:29.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.650303 23577 master_runner.cc:187] Error getting master registration for 127.23.2.189:44311: OK, instance_id { permanent_uuid: "63d2980890f64afaa34f41df2757293a" instance_seqno: 1786564827505631 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:00:29.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.676703 23577 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:00:29.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:29.742537 23602 tablet.cc:2406] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:00:29.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.759176 23851 raft_consensus.cc:493] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:00:29.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.759785 23851 raft_consensus.cc:515] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } }
20:00:29.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.762637 23851 leader_election.cc:290] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 63d2980890f64afaa34f41df2757293a (127.23.2.189:44311), 2697fd68827a42f5b8d2a7c5782df5d0 (127.23.2.188:40447)
20:00:29.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.764880 23797 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "32b9a40abcb0416ea5643975d54f6ee0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" is_pre_election: true
20:00:29.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.765846 23797 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 32b9a40abcb0416ea5643975d54f6ee0 in term 0.
20:00:29.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.767450 23600 leader_election.cc:304] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2697fd68827a42f5b8d2a7c5782df5d0, 32b9a40abcb0416ea5643975d54f6ee0; no voters:
20:00:29.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.770900 23851 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:00:29.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.771248 23851 raft_consensus.cc:493] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:00:29.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.771571 23851 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [term 0 FOLLOWER]: Advancing to term 1
20:00:29.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.783978 23851 raft_consensus.cc:515] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } }
20:00:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.791436 23797 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "32b9a40abcb0416ea5643975d54f6ee0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2697fd68827a42f5b8d2a7c5782df5d0"
20:00:29.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.791313 23851 leader_election.cc:290] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [CANDIDATE]: Term 1 election: Requested vote from peers 63d2980890f64afaa34f41df2757293a (127.23.2.189:44311), 2697fd68827a42f5b8d2a7c5782df5d0 (127.23.2.188:40447)
20:00:29.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.792027 23797 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0 [term 0 FOLLOWER]: Advancing to term 1
20:00:29.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.801113 23797 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 32b9a40abcb0416ea5643975d54f6ee0 in term 1.
20:00:29.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.802757 23600 leader_election.cc:304] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2697fd68827a42f5b8d2a7c5782df5d0, 32b9a40abcb0416ea5643975d54f6ee0; no voters:
20:00:29.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.803973 23851 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [term 1 FOLLOWER]: Leader election won for term 1
20:00:29.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.810490 23851 raft_consensus.cc:697] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [term 1 LEADER]: Becoming Leader. State: Replica: 32b9a40abcb0416ea5643975d54f6ee0, State: Running, Role: LEADER
20:00:29.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.812157 23851 consensus_queue.cc:237] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } }
20:00:29.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.835705 23866 sys_catalog.cc:455] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 32b9a40abcb0416ea5643975d54f6ee0. Latest consensus state: current_term: 1 leader_uuid: "32b9a40abcb0416ea5643975d54f6ee0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } } }
20:00:29.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.836568 23866 sys_catalog.cc:458] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [sys.catalog]: This master's current role is: LEADER
20:00:29.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.840792 23868 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
20:00:29.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.848242 23868 catalog_manager.cc:1520] Initializing Kudu cluster ID...
20:00:29.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.870363 23797 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0 [term 1 FOLLOWER]: Refusing update from remote peer 32b9a40abcb0416ea5643975d54f6ee0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:00:29.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.872648 23866 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:00:29.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.888655 23742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } attempt: 7
20:00:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.908383 23869 mvcc.cc:204] Tried to move back new op lower bound from 7317769543132262400 to 7317769542926123008. Current Snapshot: MvccSnapshot[applied={T|T < 7317769543132262400}]
20:00:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.910678 23835 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 32b9a40abcb0416ea5643975d54f6ee0. Latest consensus state: current_term: 1 leader_uuid: "32b9a40abcb0416ea5643975d54f6ee0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } } }
20:00:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.916970 23835 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0 [sys.catalog]: This master's current role is: FOLLOWER
20:00:29.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.918660 23851 sys_catalog.cc:455] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "32b9a40abcb0416ea5643975d54f6ee0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } } }
20:00:29.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.919386 23851 sys_catalog.cc:458] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [sys.catalog]: This master's current role is: LEADER
20:00:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.925570 23868 catalog_manager.cc:1383] Generated new cluster ID: 23b09cf6edfa4b34aaeb73464d345794
20:00:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.925853 23868 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
20:00:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.960565 23742 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a: Bootstrap starting.
20:00:29.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.980489 23835 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "32b9a40abcb0416ea5643975d54f6ee0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } } }
20:00:29.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:29.981287 23835 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0 [sys.catalog]: This master's current role is: FOLLOWER
20:00:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.005693 23742 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a: Neither blocks nor log segments found. Creating new log.
20:00:30.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.010571 23868 catalog_manager.cc:1406] Generated new certificate authority record
20:00:30.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.012563 23742 log.cc:826] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a: Log is configured to *not* fsync() on all Append() calls
20:00:30.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.013047 23868 catalog_manager.cc:1540] Loading token signing keys...
20:00:30.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.038525 23868 catalog_manager.cc:6092] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0: Generated new TSK 0
20:00:30.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.040977 23868 catalog_manager.cc:1550] Initializing in-progress tserver states...
20:00:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.045105 23742 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a: No bootstrap required, opened a new log
20:00:30.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.068763 23562 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:30.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:30.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-0/wal
20:00:30.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-0/data
20:00:30.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:30.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:30.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:30.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:30.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-0/logs
20:00:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-0/data/info.pb
20:00:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:00:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.129:0
20:00:30.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.129
20:00:30.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.129
20:00:30.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:30.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:43005,127.23.2.189:44311,127.23.2.188:40447
20:00:30.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:41767
20:00:30.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:30.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:30.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:00:30.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.181762 23742 raft_consensus.cc:359] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } }
20:00:30.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.184569 23742 raft_consensus.cc:385] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:00:30.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.185742 23742 raft_consensus.cc:740] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 63d2980890f64afaa34f41df2757293a, State: Initialized, Role: FOLLOWER
20:00:30.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.188815 23742 consensus_queue.cc:260] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } }
20:00:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.209336 23881 sys_catalog.cc:455] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } } }
20:00:30.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.210183 23881 sys_catalog.cc:458] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a [sys.catalog]: This master's current role is: FOLLOWER
20:00:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.216969 23712 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "32b9a40abcb0416ea5643975d54f6ee0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "63d2980890f64afaa34f41df2757293a"
20:00:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.217811 23712 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a [term 0 FOLLOWER]: Advancing to term 1
20:00:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.220305 23713 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "32b9a40abcb0416ea5643975d54f6ee0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "63d2980890f64afaa34f41df2757293a" is_pre_election: true
20:00:30.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.227955 23712 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 32b9a40abcb0416ea5643975d54f6ee0 in term 1.
20:00:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.229714 23711 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a [term 1 FOLLOWER]: Refusing update from remote peer 32b9a40abcb0416ea5643975d54f6ee0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:00:30.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.236440 23866 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:00:30.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.238729 23742 sys_catalog.cc:565] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a [sys.catalog]: configured and running, proceeding with master startup.
20:00:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.328744 23881 sys_catalog.cc:455] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 32b9a40abcb0416ea5643975d54f6ee0. Latest consensus state: current_term: 1 leader_uuid: "32b9a40abcb0416ea5643975d54f6ee0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } } }
20:00:30.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.328907 23893 sys_catalog.cc:455] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "32b9a40abcb0416ea5643975d54f6ee0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } } }
20:00:30.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.329556 23881 sys_catalog.cc:458] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a [sys.catalog]: This master's current role is: FOLLOWER
20:00:30.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.329612 23893 sys_catalog.cc:458] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a [sys.catalog]: This master's current role is: FOLLOWER
20:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.337056 23901 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:00:30.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.518985 23847 catalog_manager.cc:1295] Loaded cluster ID: 23b09cf6edfa4b34aaeb73464d345794
20:00:30.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.519778 23847 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0: loading cluster ID for follower catalog manager: success
20:00:30.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.528198 23847 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0: acquiring CA information for follower catalog manager: success
20:00:30.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.535018 23847 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 2697fd68827a42f5b8d2a7c5782df5d0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:00:30.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:30.752939 23880 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:00:30.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:30.753628 23880 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:30.755012 23880 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:00:30.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:30.815371 23880 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:00:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:30.816751 23880 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.2.129
20:00:30.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.827781 23866 sys_catalog.cc:455] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "32b9a40abcb0416ea5643975d54f6ee0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "32b9a40abcb0416ea5643975d54f6ee0" member_type: VOTER last_known_addr { host: "127.23.2.190" port: 43005 } } peers { permanent_uuid: "63d2980890f64afaa34f41df2757293a" member_type: VOTER last_known_addr { host: "127.23.2.189" port: 44311 } } peers { permanent_uuid: "2697fd68827a42f5b8d2a7c5782df5d0" member_type: VOTER last_known_addr { host: "127.23.2.188" port: 40447 } } }
20:00:30.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.828504 23866 sys_catalog.cc:458] T 00000000000000000000000000000000 P 32b9a40abcb0416ea5643975d54f6ee0 [sys.catalog]: This master's current role is: LEADER
20:00:30.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.889447 23880 tablet_server_runner.cc:78] Tablet server non-default flags:
20:00:30.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-0/logs
20:00:30.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:30.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:30.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:30.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:41767
20:00:30.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:30.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:30.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-0/data
20:00:30.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-0/wal
20:00:30.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:30.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:30.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.129:0
20:00:30.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:30.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:30.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:30.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-0/data/info.pb
20:00:30.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.129
20:00:30.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:30.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:43005,127.23.2.189:44311,127.23.2.188:40447
20:00:30.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:30.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:30.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:30.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:30.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:30.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.129
20:00:30.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:30.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:00:30.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:00:30.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 565fa2a0c8b8655dfe08c1dca95e4b4d9408fc9c
20:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 19:45:48 UTC on e7f111948823
20:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13311
20:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.891171 23880 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:30.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:30.893354 23880 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:00:30.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:30.910367 23912 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:31.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:31.376231 23901 catalog_manager.cc:1295] Loaded cluster ID: 23b09cf6edfa4b34aaeb73464d345794
20:00:31.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:31.376564 23901 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a: loading cluster ID for follower catalog manager: success
20:00:31.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:31.407868 23901 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a: acquiring CA information for follower catalog manager: success
20:00:31.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:31.446455 23901 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 63d2980890f64afaa34f41df2757293a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:00:33.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:32.310482 23911 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee40 after lost signal to thread 23880
20:00:33.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:33.624634 23911 kernel_stack_watchdog.cc:198] Thread 23880 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 397ms:
20:00:33.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:00:33.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:00:33.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:00:33.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:00:33.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:33.625605 23880 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.714s user 0.865s sys 1.280s
20:00:33.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:33.625969 23880 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.714s user 0.865s sys 1.280s
20:00:33.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:33.627243 23915 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2715 milliseconds
20:00:33.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:30.912879 23913 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:33.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:33.627974 23916 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:33.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.628464 23880 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:00:33.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.632680 23880 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:33.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.644059 23880 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:33.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.645547 23880 hybrid_clock.cc:648] HybridClock initialized: now 1786564833645492 us; error 71 us; skew 500 ppm
20:00:33.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.646672 23880 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:33.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.672915 23880 webserver.cc:533] Webserver started at http://127.23.2.129:33537/ using document root <none> and password file <none>
20:00:33.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.674212 23880 fs_manager.cc:362] Metadata directory not provided
20:00:33.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.674496 23880 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.675074 23880 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:00:33.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.681550 23880 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-0/data/instance:
20:00:33.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "36c3fca7ac664148b823b7149b87b476"
20:00:33.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:33 on dist-test-slave-1jjb"
20:00:33.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.683046 23880 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-0/wal/instance:
20:00:33.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "36c3fca7ac664148b823b7149b87b476"
20:00:33.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:33 on dist-test-slave-1jjb"
20:00:33.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.698150 23880 fs_manager.cc:696] Time spent creating directory manager: real 0.014s user 0.004s sys 0.012s
20:00:33.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.711575 23926 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:33.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.713173 23880 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.006s sys 0.004s
20:00:33.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.713586 23880 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-0/data,/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-0/wal
20:00:33.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "36c3fca7ac664148b823b7149b87b476"
20:00:33.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:33 on dist-test-slave-1jjb"
20:00:33.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.714035 23880 fs_report.cc:389] FS layout report
20:00:33.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:33.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-0/wal
20:00:33.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-0/wal
20:00:33.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-0/data/data
20:00:33.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:33.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:33.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:33.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:33.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:33.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:33.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:33.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:33.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:33.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:33.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.794907 23880 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:00:33.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.796583 23880 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:33.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.797011 23880 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:33.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.799906 23880 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:00:33.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.805016 23880 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:00:33.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.805258 23880 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:00:33.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.805563 23880 ts_tablet_manager.cc:616] Registered 0 tablets
20:00:33.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.805783 23880 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:00:33.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.990705 23880 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.129:34237
20:00:33.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.990901 24059 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.2.129:34237 every 8 connection(s)
20:00:33.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:33.993566 23880 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-0/data/info.pb
20:00:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.002337 23562 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu as pid 23880
20:00:34.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.003000 23562 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-0/wal/instance
20:00:34.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.022970 23562 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:34.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:34.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-1/wal
20:00:34.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-1/data
20:00:34.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-1/logs
20:00:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-1/data/info.pb
20:00:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:00:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.130:0
20:00:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.130
20:00:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.130
20:00:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:43005,127.23.2.189:44311,127.23.2.188:40447
20:00:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:41767
20:00:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:00:34.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.066133 24063 heartbeater.cc:344] Connected to a master server at 127.23.2.190:43005
20:00:34.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.066663 24063 heartbeater.cc:461] Registering TS with master...
20:00:34.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.068037 24063 heartbeater.cc:507] Master 127.23.2.190:43005 requested a full tablet report, sending...
20:00:34.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.071264 23613 ts_manager.cc:194] Registered new tserver with Master: 36c3fca7ac664148b823b7149b87b476 (127.23.2.129:34237)
20:00:34.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.074666 23613 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.2.129:41229
20:00:34.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.094348 24061 heartbeater.cc:344] Connected to a master server at 127.23.2.189:44311
20:00:34.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.095209 24061 heartbeater.cc:461] Registering TS with master...
20:00:34.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.095886 24061 heartbeater.cc:507] Master 127.23.2.189:44311 requested a full tablet report, sending...
20:00:34.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.099536 23703 ts_manager.cc:194] Registered new tserver with Master: 36c3fca7ac664148b823b7149b87b476 (127.23.2.129:34237)
20:00:34.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.107290 24060 heartbeater.cc:344] Connected to a master server at 127.23.2.188:40447
20:00:34.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.107676 24060 heartbeater.cc:461] Registering TS with master...
20:00:34.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.108330 24060 heartbeater.cc:507] Master 127.23.2.188:40447 requested a full tablet report, sending...
20:00:34.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.110867 23785 ts_manager.cc:194] Registered new tserver with Master: 36c3fca7ac664148b823b7149b87b476 (127.23.2.129:34237)
20:00:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:34.440724 24072 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:00:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:34.441395 24072 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:34.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:34.442003 24072 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:00:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:34.483116 24072 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:00:34.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:34.485257 24072 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.2.130
20:00:34.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.529721 24072 tablet_server_runner.cc:78] Tablet server non-default flags:
20:00:34.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-1/logs
20:00:34.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:34.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:34.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:34.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:41767
20:00:34.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:34.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:34.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-1/data
20:00:34.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-1/wal
20:00:34.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:34.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:34.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.130:0
20:00:34.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:34.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:34.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-1/data/info.pb
20:00:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.130
20:00:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:43005,127.23.2.189:44311,127.23.2.188:40447
20:00:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.130
20:00:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:00:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:00:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 565fa2a0c8b8655dfe08c1dca95e4b4d9408fc9c
20:00:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:34.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 19:45:48 UTC on e7f111948823
20:00:34.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13311
20:00:34.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:34.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.531509 24072 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:34.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:34.533771 24072 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:00:34.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:34.586694 24079 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:35.079737 24063 heartbeater.cc:499] Master 127.23.2.190:43005 was elected leader, sending a full tablet report...
20:00:36.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:36.271317 23738 debug-util.cc:398] Leaking SignalData structure 0x7b08000811c0 after lost signal to thread 23661
20:00:36.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:36.272487 23738 debug-util.cc:398] Leaking SignalData structure 0x7b0800098280 after lost signal to thread 23741
20:00:36.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:34.591158 24080 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:36.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:35.991606 24077 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee40 after lost signal to thread 24072
20:00:36.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:36.279843 24072 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.689s user 0.589s sys 1.079s
20:00:36.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:36.281406 24072 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.691s user 0.589s sys 1.079s
20:00:36.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:36.282543 24082 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:36.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:36.285444 24081 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1687 milliseconds
20:00:36.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.285493 24072 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:00:36.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.287253 24072 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:36.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.290118 24072 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:36.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.291575 24072 hybrid_clock.cc:648] HybridClock initialized: now 1786564836291444 us; error 125 us; skew 500 ppm
20:00:36.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.292781 24072 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.301055 24072 webserver.cc:533] Webserver started at http://127.23.2.130:43079/ using document root <none> and password file <none>
20:00:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.302351 24072 fs_manager.cc:362] Metadata directory not provided
20:00:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.302634 24072 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:36.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.303237 24072 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:00:36.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.310493 24072 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-1/data/instance:
20:00:36.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff3b48187e7246b8b204825ba25deee4"
20:00:36.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:36 on dist-test-slave-1jjb"
20:00:36.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.312110 24072 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-1/wal/instance:
20:00:36.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff3b48187e7246b8b204825ba25deee4"
20:00:36.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:36 on dist-test-slave-1jjb"
20:00:36.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.322420 24072 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.005s sys 0.004s
20:00:36.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.328608 24090 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:36.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.329929 24072 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
20:00:36.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.330242 24072 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-1/data,/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-1/wal
20:00:36.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff3b48187e7246b8b204825ba25deee4"
20:00:36.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:36 on dist-test-slave-1jjb"
20:00:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.330556 24072 fs_report.cc:389] FS layout report
20:00:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-1/wal
20:00:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-1/wal
20:00:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-1/data/data
20:00:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:36.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.397053 24072 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:00:36.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.398752 24072 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:36.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.399290 24072 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:36.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.404376 24072 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:00:36.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.408885 24072 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:00:36.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.409134 24072 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:00:36.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.409466 24072 ts_tablet_manager.cc:616] Registered 0 tablets
20:00:36.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.409667 24072 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:00:36.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.590179 24072 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.130:39951
20:00:36.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.590391 24222 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.2.130:39951 every 8 connection(s)
20:00:36.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.594182 24072 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-1/data/info.pb
20:00:36.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.599298 23562 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu as pid 24072
20:00:36.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.599781 23562 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-1/wal/instance
20:00:36.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.612694 23562 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:36.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:36.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-2/wal
20:00:36.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-2/data
20:00:36.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:36.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:36.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:36.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:36.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:36.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:36.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-2/logs
20:00:36.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-2/data/info.pb
20:00:36.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:36.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:36.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:36.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:36.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:36.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:36.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:36.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:36.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:00:36.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:36.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.131:0
20:00:36.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.131
20:00:36.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.131
20:00:36.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:36.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:43005,127.23.2.189:44311,127.23.2.188:40447
20:00:36.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:41767
20:00:36.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:00:36.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.658115 24225 heartbeater.cc:344] Connected to a master server at 127.23.2.188:40447
20:00:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.658627 24225 heartbeater.cc:461] Registering TS with master...
20:00:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.660027 24225 heartbeater.cc:507] Master 127.23.2.188:40447 requested a full tablet report, sending...
20:00:36.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.664296 23785 ts_manager.cc:194] Registered new tserver with Master: ff3b48187e7246b8b204825ba25deee4 (127.23.2.130:39951)
20:00:36.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.669305 24226 heartbeater.cc:344] Connected to a master server at 127.23.2.189:44311
20:00:36.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.669697 24226 heartbeater.cc:461] Registering TS with master...
20:00:36.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.670428 24229 heartbeater.cc:344] Connected to a master server at 127.23.2.190:43005
20:00:36.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.670715 24226 heartbeater.cc:507] Master 127.23.2.189:44311 requested a full tablet report, sending...
20:00:36.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.670840 24229 heartbeater.cc:461] Registering TS with master...
20:00:36.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.671880 24229 heartbeater.cc:507] Master 127.23.2.190:43005 requested a full tablet report, sending...
20:00:36.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.673781 23703 ts_manager.cc:194] Registered new tserver with Master: ff3b48187e7246b8b204825ba25deee4 (127.23.2.130:39951)
20:00:36.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.674404 23613 ts_manager.cc:194] Registered new tserver with Master: ff3b48187e7246b8b204825ba25deee4 (127.23.2.130:39951)
20:00:36.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:36.676362 23613 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.2.130:36785
20:00:36.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:36.983603 24232 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:00:36.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:36.984167 24232 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:36.984813 24232 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:00:37.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:37.027477 24232 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:00:37.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:37.028533 24232 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.2.131
20:00:37.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:37.068605 24232 tablet_server_runner.cc:78] Tablet server non-default flags:
20:00:37.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-2/logs
20:00:37.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:37.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:37.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:37.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.2.148:41767
20:00:37.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:37.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:37.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-2/data
20:00:37.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-2/wal
20:00:37.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.2.131:0
20:00:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-2/data/info.pb
20:00:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.2.131
20:00:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.2.190:43005,127.23.2.189:44311,127.23.2.188:40447
20:00:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.2.131
20:00:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:00:37.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:00:37.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 565fa2a0c8b8655dfe08c1dca95e4b4d9408fc9c
20:00:37.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:37.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 19:45:48 UTC on e7f111948823
20:00:37.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13311
20:00:37.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:37.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:37.070482 24232 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:37.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:37.072851 24232 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:00:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:37.098444 24242 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:37.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:37.680379 24229 heartbeater.cc:499] Master 127.23.2.190:43005 was elected leader, sending a full tablet report...
20:00:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:37.103622 24243 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:39.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:38.998376 24232 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.893s user 0.001s sys 0.000s
20:00:39.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:39.001142 24232 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.896s user 0.002s sys 0.000s
20:00:39.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:38.498651 24241 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee40 after lost signal to thread 24232
20:00:39.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:39.002045 24246 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:39.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:39.009131 24244 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1905 milliseconds
20:00:39.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.009231 24232 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:00:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.010772 24232 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.013485 24232 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.015110 24232 hybrid_clock.cc:648] HybridClock initialized: now 1786564839015041 us; error 100 us; skew 500 ppm
20:00:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.016278 24232 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:39.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.024879 24232 webserver.cc:533] Webserver started at http://127.23.2.131:35429/ using document root <none> and password file <none>
20:00:39.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.025852 24232 fs_manager.cc:362] Metadata directory not provided
20:00:39.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.026101 24232 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:39.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.026659 24232 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:00:39.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.031625 24232 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-2/data/instance:
20:00:39.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "105cb0c3d708460ebfcd1505f8809fde"
20:00:39.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:39 on dist-test-slave-1jjb"
20:00:39.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.032841 24232 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-2/wal/instance:
20:00:39.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "105cb0c3d708460ebfcd1505f8809fde"
20:00:39.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:39 on dist-test-slave-1jjb"
20:00:39.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.041175 24232 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
20:00:39.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.047405 24255 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:39.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.048666 24232 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
20:00:39.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.049029 24232 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-2/data,/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-2/wal
20:00:39.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "105cb0c3d708460ebfcd1505f8809fde"
20:00:39.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:39 on dist-test-slave-1jjb"
20:00:39.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.049401 24232 fs_report.cc:389] FS layout report
20:00:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-2/wal
20:00:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-2/wal
20:00:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-2/data/data
20:00:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:39.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.107008 24232 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:00:39.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.108649 24232 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:39.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.109212 24232 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:39.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.111965 24232 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:00:39.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.116237 24232 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:00:39.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.116467 24232 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:00:39.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.116812 24232 ts_tablet_manager.cc:616] Registered 0 tablets
20:00:39.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.117116 24232 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:00:39.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.262645 24232 rpc_server.cc:307] RPC server started. Bound to: 127.23.2.131:38955
20:00:39.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.262771 24399 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.2.131:38955 every 8 connection(s)
20:00:39.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.265311 24232 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-2/data/info.pb
20:00:39.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.272863 23562 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu as pid 24232
20:00:39.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.273422 23562 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster3679423216767831184/ts-2/wal/instance
20:00:39.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.334471 24401 heartbeater.cc:344] Connected to a master server at 127.23.2.189:44311
20:00:39.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.334596 24402 heartbeater.cc:344] Connected to a master server at 127.23.2.190:43005
20:00:39.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.335040 24402 heartbeater.cc:461] Registering TS with master...
20:00:39.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.335464 24400 heartbeater.cc:344] Connected to a master server at 127.23.2.188:40447
20:00:39.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.335035 24401 heartbeater.cc:461] Registering TS with master...
20:00:39.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.335793 24400 heartbeater.cc:461] Registering TS with master...
20:00:39.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.336510 24402 heartbeater.cc:507] Master 127.23.2.190:43005 requested a full tablet report, sending...
20:00:39.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.336833 24401 heartbeater.cc:507] Master 127.23.2.189:44311 requested a full tablet report, sending...
20:00:39.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.336881 24400 heartbeater.cc:507] Master 127.23.2.188:40447 requested a full tablet report, sending...
20:00:39.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.339285 23613 ts_manager.cc:194] Registered new tserver with Master: 105cb0c3d708460ebfcd1505f8809fde (127.23.2.131:38955)
20:00:39.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.339764 23703 ts_manager.cc:194] Registered new tserver with Master: 105cb0c3d708460ebfcd1505f8809fde (127.23.2.131:38955)
20:00:39.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.340227 23784 ts_manager.cc:194] Registered new tserver with Master: 105cb0c3d708460ebfcd1505f8809fde (127.23.2.131:38955)
20:00:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.340996 23613 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.2.131:39669
20:00:39.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:39.348603 23562 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
20:00:39.360 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:00:39.362 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:00:39.371 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.23.2.190"
port: 43005
}
bound_http_address {
host: "127.23.2.190"
port: 35881
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.23.2.189"
port: 44311
}
bound_http_address {
host: "127.23.2.189"
port: 44673
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.23.2.188"
port: 40447
}
bound_http_address {
host: "127.23.2.188"
port: 46625
}
}
}
20:00:39.374 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:00:39.377 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.23.2.129"
port: 34237
}
bound_http_address {
host: "127.23.2.129"
port: 33537
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.23.2.130"
port: 39951
}
bound_http_address {
host: "127.23.2.130"
port: 43079
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.23.2.131"
port: 38955
}
bound_http_address {
host: "127.23.2.131"
port: 35429
}
}
}
20:00:39.377 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:00:39.551 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
20:00:39.607 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:00:39.719 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskwpaMk5/test-tmp
20:00:39.719 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:00:39.720 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskwpaMk5/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:00:39.724 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
20:00:40.084 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:00:40.085 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:00:40.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:40.090441 24415 mini_oidc.cc:150] Starting JWKS server
20:00:40.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:40.093811 24415 webserver.cc:533] Webserver started at http://127.0.0.1:34355/ using document root <none> and password file <none>
20:00:40.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:40.094219 24415 mini_oidc.cc:186] Starting OIDC Discovery server
20:00:40.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:40.095993 24415 webserver.cc:533] Webserver started at http://127.0.0.1:44719/ using document root <none> and password file <none>
20:00:40.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
20:00:40.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/principal' for realm 'KRBTEST.COM',
20:00:40.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
20:00:40.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:40 dist-test-slave-1jjb krb5kdc[24428](info): setting up network...
20:00:40.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
20:00:40.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:40 dist-test-slave-1jjb krb5kdc[24428](info): set up 2 sockets
20:00:40.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:40 dist-test-slave-1jjb krb5kdc[24428](info): commencing operation
20:00:40.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
20:00:40.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:40.344318 24402 heartbeater.cc:499] Master 127.23.2.190:43005 was elected leader, sending a full tablet report...
20:00:42.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:42.507983 24415 mini_kdc.cc:121] Time spent starting KDC: real 2.373s user 0.005s sys 0.016s
20:00:42.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
20:00:42.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:00:42.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
20:00:42.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
20:00:42.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:00:42.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
20:00:42.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
20:00:42.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:00:42.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
20:00:42.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:00:42.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/test-user.keytab.
20:00:42.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/test-user.keytab.
20:00:42.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:42 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564842, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:00:42.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:00:42.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T20:00:42Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:00:42.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T20:00:42Z Disabled control of system clock
20:00:42.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.23.215.254@KRBTEST.COM; defaulting to no policy
20:00:42.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:42.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.23.215.254@KRBTEST.COM" created.
20:00:42.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:42.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.215.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:42.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.215.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:42.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.23.215.254@KRBTEST.COM; defaulting to no policy
20:00:42.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:42.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.23.215.254@KRBTEST.COM" created.
20:00:42.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:42.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.215.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:42.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.215.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:42.740849 24415 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/wal
20:00:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/data
20:00:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/logs
20:00:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/data/info.pb
20:00:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:42.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:42.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:42.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:42.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:42.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:42.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:42.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:00:42.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:42.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:42.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:42.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.215.254:39917
20:00:42.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.215.254
20:00:42.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:42.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab
20:00:42.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.215.254
20:00:42.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:00:42.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:00:42.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:00:42.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:00:42.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.215.212:40571
20:00:42.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:42.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:42.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:42.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:42.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057
20:00:42.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:42.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34355/jwks.json
20:00:42.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:00:42.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:00:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:43.079799 24450 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:00:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:43.080420 24450 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:43.081008 24450 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:00:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:43.114113 24450 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:00:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:43.114344 24450 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:00:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:43.114638 24450 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:00:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:43.114826 24450 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:00:43.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:43.115029 24450 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:00:43.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:43.115226 24450 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:00:43.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:43.115381 24450 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34355/jwks.json
20:00:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:43.156953 24450 master_runner.cc:387] Master server non-default flags:
20:00:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/logs
20:00:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.215.212:40571
20:00:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/data
20:00:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/wal
20:00:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:00:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.215.254
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.215.254:39917
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34355/jwks.json
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/data/info.pb
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.215.254
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:43.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:43.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:43.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:00:43.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:00:43.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 565fa2a0c8b8655dfe08c1dca95e4b4d9408fc9c
20:00:43.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:43.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 19:45:48 UTC on e7f111948823
20:00:43.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13311
20:00:43.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:43.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:43.158382 24450 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:43.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:43.159438 24450 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:00:43.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:43.170853 24459 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:44.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:43.172640 24460 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:44.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:44.700623 24461 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1528 milliseconds
20:00:44.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:44.701093 24462 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:44.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.701129 24450 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:00:44.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.704926 24450 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:44.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.707469 24450 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:44.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.708811 24450 hybrid_clock.cc:648] HybridClock initialized: now 1786564844708768 us; error 121 us; skew 500 ppm
20:00:44.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.709649 24450 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:44.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:44 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564844, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:00:44.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.716166 24450 init.cc:377] Logged in from keytab as kudu/127.23.215.254@KRBTEST.COM (short username kudu)
20:00:44.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.721662 24450 webserver.cc:533] Webserver started at http://127.23.215.254:46357/ using document root <none> and password file <none>
20:00:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.722656 24450 fs_manager.cc:362] Metadata directory not provided
20:00:44.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.722906 24450 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:44.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.723456 24450 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:00:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.727826 24450 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/data/instance:
20:00:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2882c6717446405899962890e037a3db"
20:00:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:44 on dist-test-slave-1jjb"
20:00:44.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.729105 24450 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/wal/instance:
20:00:44.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2882c6717446405899962890e037a3db"
20:00:44.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:44 on dist-test-slave-1jjb"
20:00:44.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.737291 24450 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
20:00:44.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.742983 24475 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:44.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.743992 24450 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
20:00:44.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.744405 24450 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/data,/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/wal
20:00:44.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2882c6717446405899962890e037a3db"
20:00:44.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:44 on dist-test-slave-1jjb"
20:00:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.744763 24450 fs_report.cc:389] FS layout report
20:00:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/wal
20:00:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/wal
20:00:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/data/data
20:00:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:44.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:44.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:44.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:44.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:44.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.805593 24450 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:00:44.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.818563 24450 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:44.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.819094 24450 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:44.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.899682 24450 rpc_server.cc:307] RPC server started. Bound to: 127.23.215.254:39917
20:00:44.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.899744 24537 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.215.254:39917 every 8 connection(s)
20:00:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.902943 24450 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/data/info.pb
20:00:44.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.908833 24415 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu as pid 24450
20:00:44.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.909276 24538 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:00:44.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.909353 24415 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/wal/instance
20:00:44.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.921360 24538 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } has no permanent_uuid. Determining permanent_uuid...
20:00:44.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.23.215.253@KRBTEST.COM; defaulting to no policy
20:00:44.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:44.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.23.215.253@KRBTEST.COM" created.
20:00:44.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:44 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564844, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.254@KRBTEST.COM for kudu/127.23.215.254@KRBTEST.COM
20:00:44.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:44.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.215.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:44.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.215.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:44.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:44.967880 24538 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } has no permanent_uuid. Determining permanent_uuid...
20:00:44.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:44.970512 24480 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.215.253:36309: connect: Connection refused (error 111)
20:00:44.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:44.974694 24538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.253:36309: Network error: Client connection negotiation failed: client connection to 127.23.215.253:36309: connect: Connection refused (error 111)
20:00:44.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.23.215.253@KRBTEST.COM; defaulting to no policy
20:00:44.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:44.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.23.215.253@KRBTEST.COM" created.
20:00:44.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:45.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.215.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:45.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.215.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:45.001847 24415 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/wal
20:00:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/data
20:00:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/logs
20:00:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/data/info.pb
20:00:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.215.253:36309
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.215.253
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.215.253
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.215.212:40571
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34355/jwks.json
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:00:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:00:45.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:45.024214 24538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } attempt: 1
20:00:45.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:45.028550 24538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.253:36309: Network error: Client connection negotiation failed: client connection to 127.23.215.253:36309: connect: Connection refused (error 111)
20:00:45.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:45.096967 24538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } attempt: 2
20:00:45.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:45.101405 24538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.253:36309: Network error: Client connection negotiation failed: client connection to 127.23.215.253:36309: connect: Connection refused (error 111)
20:00:45.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:45.192888 24538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } attempt: 3
20:00:45.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:45.197408 24538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.253:36309: Network error: Client connection negotiation failed: client connection to 127.23.215.253:36309: connect: Connection refused (error 111)
20:00:45.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:45.333837 24548 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:00:45.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:45.334373 24548 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:45.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:45.334944 24548 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:00:45.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:45.340832 24538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } attempt: 4
20:00:45.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:45.344986 24538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.253:36309: Network error: Client connection negotiation failed: client connection to 127.23.215.253:36309: connect: Connection refused (error 111)
20:00:45.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:45.369962 24548 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:00:45.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:45.370581 24548 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:00:45.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:45.371235 24548 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:00:45.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:45.371605 24548 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:00:45.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:45.372241 24548 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:00:45.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:45.372838 24548 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:00:45.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:45.373273 24548 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34355/jwks.json
20:00:45.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:45.412254 24548 master_runner.cc:387] Master server non-default flags:
20:00:45.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/logs
20:00:45.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:45.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:45.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:45.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.215.212:40571
20:00:45.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:45.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/data
20:00:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/wal
20:00:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:00:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057
20:00:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab
20:00:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.215.253
20:00:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.215.253:36309
20:00:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34355/jwks.json
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/data/info.pb
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.215.253
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 565fa2a0c8b8655dfe08c1dca95e4b4d9408fc9c
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 19:45:48 UTC on e7f111948823
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13311
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:45.413540 24548 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:45.414597 24548 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:00:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:45.425971 24554 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:45.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:45.644469 24538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } attempt: 5
20:00:45.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:45.653383 24538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.253:36309: Network error: Client connection negotiation failed: client connection to 127.23.215.253:36309: connect: Connection refused (error 111)
20:00:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:46.200942 24538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } attempt: 6
20:00:46.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:46.225757 24538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.253:36309: Network error: Client connection negotiation failed: client connection to 127.23.215.253:36309: connect: Connection refused (error 111)
20:00:47.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:45.426535 24555 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:47.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.252954 24548 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.827s user 0.659s sys 1.119s
20:00:47.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.253669 24548 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.827s user 0.659s sys 1.119s
20:00:47.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:46.830464 24553 debug-util.cc:398] Leaking SignalData structure 0x7b0800034fe0 after lost signal to thread 24548
20:00:47.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.255513 24557 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:47.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.259478 24556 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1832 milliseconds
20:00:47.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.259534 24548 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:00:47.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.261157 24548 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:47.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.264392 24548 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:47.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.265987 24548 hybrid_clock.cc:648] HybridClock initialized: now 1786564847265940 us; error 39 us; skew 500 ppm
20:00:47.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.267055 24548 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:47.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:47 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564847, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:00:47.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.274708 24548 init.cc:377] Logged in from keytab as kudu/127.23.215.253@KRBTEST.COM (short username kudu)
20:00:47.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.280237 24548 webserver.cc:533] Webserver started at http://127.23.215.253:37699/ using document root <none> and password file <none>
20:00:47.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.281193 24548 fs_manager.cc:362] Metadata directory not provided
20:00:47.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.281399 24548 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:47.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.281893 24548 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:00:47.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.286345 24538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } attempt: 7
20:00:47.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.286278 24548 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/data/instance:
20:00:47.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "14182101312f479992e2922ed266d990"
20:00:47.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:47 on dist-test-slave-1jjb"
20:00:47.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.287518 24548 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/wal/instance:
20:00:47.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "14182101312f479992e2922ed266d990"
20:00:47.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:47 on dist-test-slave-1jjb"
20:00:47.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.293217 24538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.253:36309: Network error: Client connection negotiation failed: client connection to 127.23.215.253:36309: connect: Connection refused (error 111)
20:00:47.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.296825 24548 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
20:00:47.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.303280 24572 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:47.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.304543 24548 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
20:00:47.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.304836 24548 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/data,/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/wal
20:00:47.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "14182101312f479992e2922ed266d990"
20:00:47.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:47 on dist-test-slave-1jjb"
20:00:47.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.305130 24548 fs_report.cc:389] FS layout report
20:00:47.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:47.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/wal
20:00:47.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/wal
20:00:47.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/data/data
20:00:47.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:47.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:47.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:47.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:47.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:47.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:47.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:47.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:47.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:47.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:47.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.361056 24548 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:00:47.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.368465 24548 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:47.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.369282 24548 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:47.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.454275 24548 rpc_server.cc:307] RPC server started. Bound to: 127.23.215.253:36309
20:00:47.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.454350 24636 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.215.253:36309 every 8 connection(s)
20:00:47.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.457646 24548 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/data/info.pb
20:00:47.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.463503 24415 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu as pid 24548
20:00:47.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.464030 24415 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/wal/instance
20:00:47.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.464381 24637 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:00:47.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.478870 24637 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } has no permanent_uuid. Determining permanent_uuid...
20:00:47.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.23.215.252@KRBTEST.COM; defaulting to no policy
20:00:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.23.215.252@KRBTEST.COM" created.
20:00:47.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:47 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564847, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.253@KRBTEST.COM for kudu/127.23.215.254@KRBTEST.COM
20:00:47.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.518863 24637 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } has no permanent_uuid. Determining permanent_uuid...
20:00:47.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:47.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.215.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:47.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.215.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:47.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:47 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564847, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.253@KRBTEST.COM for kudu/127.23.215.253@KRBTEST.COM
20:00:47.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.23.215.252@KRBTEST.COM; defaulting to no policy
20:00:47.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:47.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.23.215.252@KRBTEST.COM" created.
20:00:47.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.545789 24637 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } has no permanent_uuid. Determining permanent_uuid...
20:00:47.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.548777 24577 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:00:47.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.554050 24637 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.252:40057: Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:00:47.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:47.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.215.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:47.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.215.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:47.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.564389 24415 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:47.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:47.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/wal
20:00:47.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/data
20:00:47.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:47.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:47.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:47.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:47.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/logs
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/data/info.pb
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.215.252:40057
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.215.252
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.215.252
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.215.212:40571
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:47.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:47.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:47.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057
20:00:47.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:47.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34355/jwks.json
20:00:47.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:00:47.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:00:47.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.603554 24637 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } attempt: 1
20:00:47.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.608754 24637 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.252:40057: Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:00:47.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.677181 24637 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } attempt: 2
20:00:47.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.681998 24637 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.252:40057: Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:00:47.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.773402 24637 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } attempt: 3
20:00:47.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.777941 24637 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.252:40057: Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:00:47.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.892033 24649 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:00:47.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.892814 24649 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:47.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.893554 24649 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:00:47.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.921362 24637 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } attempt: 4
20:00:47.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.925807 24637 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.252:40057: Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:00:47.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.930294 24649 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:00:47.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.930588 24649 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:00:47.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.931035 24649 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:00:47.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.931360 24649 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:00:47.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.931695 24649 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:00:47.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.932019 24649 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:00:47.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.932332 24649 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34355/jwks.json
20:00:47.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.971347 24649 master_runner.cc:387] Master server non-default flags:
20:00:47.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/logs
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.215.212:40571
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/data
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/wal
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.215.252
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.215.252:40057
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34355/jwks.json
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:00:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:00:47.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:47.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:47.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/data/info.pb
20:00:47.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.215.252
20:00:47.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:47.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:47.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:47.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:47.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:47.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:47.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:47.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:00:47.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:00:47.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 565fa2a0c8b8655dfe08c1dca95e4b4d9408fc9c
20:00:47.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:47.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 19:45:48 UTC on e7f111948823
20:00:47.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13311
20:00:47.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:47.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.973102 24649 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:47.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:47.974495 24649 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:00:47.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.985848 24658 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:48.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:48.225238 24637 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } attempt: 5
20:00:48.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:48.234644 24637 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.252:40057: Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:00:48.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:48.782215 24637 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } attempt: 6
20:00:48.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:48.822084 24637 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.252:40057: Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:00:49.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.383714 24538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } attempt: 8
20:00:49.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:49 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564844, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.254@KRBTEST.COM for kudu/127.23.215.253@KRBTEST.COM
20:00:49.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.493076 24538 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } has no permanent_uuid. Determining permanent_uuid...
20:00:49.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:49.504992 24538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.252:40057: Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:00:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.570559 24538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } attempt: 1
20:00:49.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:49.578931 24538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.252:40057: Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:00:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.632512 24538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } attempt: 2
20:00:49.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:49.647137 24538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.252:40057: Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:00:49.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.723740 24538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } attempt: 3
20:00:49.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:49.732507 24538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.252:40057: Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:00:49.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:49.399780 24657 debug-util.cc:398] Leaking SignalData structure 0x7b0800034fe0 after lost signal to thread 24649
20:00:49.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:49.848342 24657 kernel_stack_watchdog.cc:198] Thread 24649 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 410ms:
20:00:49.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:00:49.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:00:49.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:00:49.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:00:49.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:47.986953 24660 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:49.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:49.849803 24649 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.863s user 0.565s sys 1.189s
20:00:49.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:49.850126 24649 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.864s user 0.565s sys 1.189s
20:00:49.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:49.852064 24663 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:49.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.856354 24649 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:00:49.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:49.856375 24661 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1868 milliseconds
20:00:49.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.857736 24649 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.860347 24649 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:49.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.861737 24649 hybrid_clock.cc:648] HybridClock initialized: now 1786564849861702 us; error 65 us; skew 500 ppm
20:00:49.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.862597 24649 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:49.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:49 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564849, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:00:49.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.869254 24649 init.cc:377] Logged in from keytab as kudu/127.23.215.252@KRBTEST.COM (short username kudu)
20:00:49.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.874255 24649 webserver.cc:533] Webserver started at http://127.23.215.252:34117/ using document root <none> and password file <none>
20:00:49.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.875195 24649 fs_manager.cc:362] Metadata directory not provided
20:00:49.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.875401 24649 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:49.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.875870 24649 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:00:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.880293 24649 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/data/instance:
20:00:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "426096099be0496fbc843c8b48c09fa4"
20:00:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:49 on dist-test-slave-1jjb"
20:00:49.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.881743 24649 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/wal/instance:
20:00:49.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "426096099be0496fbc843c8b48c09fa4"
20:00:49.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:49 on dist-test-slave-1jjb"
20:00:49.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.882652 24637 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } attempt: 7
20:00:49.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.888036 24538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } attempt: 4
20:00:49.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:49.890726 24637 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.252:40057: Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:00:49.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.891613 24649 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.005s sys 0.005s
20:00:49.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:49.893167 24538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.215.252:40057: Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:00:49.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.898455 24684 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:49.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.899639 24649 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.001s
20:00:49.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.899940 24649 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/data,/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/wal
20:00:49.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "426096099be0496fbc843c8b48c09fa4"
20:00:49.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:49 on dist-test-slave-1jjb"
20:00:49.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.900309 24649 fs_report.cc:389] FS layout report
20:00:49.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:49.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/wal
20:00:49.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/wal
20:00:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/data/data
20:00:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:49.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.954564 24649 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:00:49.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.964689 24649 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:49.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:49.965101 24649 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.084067 24649 rpc_server.cc:307] RPC server started. Bound to: 127.23.215.252:40057
20:00:50.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.084945 24744 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.215.252:40057 every 8 connection(s)
20:00:50.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.090058 24649 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/data/info.pb
20:00:50.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.093816 24415 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu as pid 24649
20:00:50.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.095429 24415 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/wal/instance
20:00:50.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.152735 24746 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:00:50.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:50 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564842, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.23.215.254@KRBTEST.COM
20:00:50.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.173923 24746 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } has no permanent_uuid. Determining permanent_uuid...
20:00:50.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.189549 24538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } attempt: 5
20:00:50.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:50 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564842, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.23.215.253@KRBTEST.COM
20:00:50.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:50 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564844, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.254@KRBTEST.COM for kudu/127.23.215.252@KRBTEST.COM
20:00:50.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:50 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564849, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.252@KRBTEST.COM for kudu/127.23.215.254@KRBTEST.COM
20:00:50.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:50 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564842, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.23.215.252@KRBTEST.COM
20:00:50.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.256312 24746 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } has no permanent_uuid. Determining permanent_uuid...
20:00:50.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.270426 24538 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db: Bootstrap starting.
20:00:50.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:50 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564849, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.252@KRBTEST.COM for kudu/127.23.215.253@KRBTEST.COM
20:00:50.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.289623 24538 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db: Neither blocks nor log segments found. Creating new log.
20:00:50.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.294729 24538 log.cc:826] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db: Log is configured to *not* fsync() on all Append() calls
20:00:50.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.307174 24746 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } has no permanent_uuid. Determining permanent_uuid...
20:00:50.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.311646 24538 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db: No bootstrap required, opened a new log
20:00:50.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:50 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564849, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.252@KRBTEST.COM for kudu/127.23.215.252@KRBTEST.COM
20:00:50.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.358842 24538 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:00:50.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.359411 24746 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4: Bootstrap starting.
20:00:50.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.359699 24538 raft_consensus.cc:385] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:00:50.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.359988 24538 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2882c6717446405899962890e037a3db, State: Initialized, Role: FOLLOWER
20:00:50.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.361601 24538 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:00:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.379112 24746 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4: Neither blocks nor log segments found. Creating new log.
20:00:50.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.381711 24757 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:00:50.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.383914 24757 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [sys.catalog]: This master's current role is: FOLLOWER
20:00:50.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.384572 24538 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [sys.catalog]: configured and running, proceeding with master startup.
20:00:50.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.385924 24746 log.cc:826] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4: Log is configured to *not* fsync() on all Append() calls
20:00:50.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.409013 24746 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4: No bootstrap required, opened a new log
20:00:50.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.460294 24746 raft_consensus.cc:359] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:00:50.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.461728 24746 raft_consensus.cc:385] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:00:50.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.462114 24746 raft_consensus.cc:740] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 426096099be0496fbc843c8b48c09fa4, State: Initialized, Role: FOLLOWER
20:00:50.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:50.460353 24055 debug-util.cc:398] Leaking SignalData structure 0x7b08000b3a00 after lost signal to thread 23902
20:00:50.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:50.461858 24055 debug-util.cc:398] Leaking SignalData structure 0x7b08000b37a0 after lost signal to thread 24059
20:00:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.463673 24746 consensus_queue.cc:260] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:00:50.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.494449 24746 sys_catalog.cc:565] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [sys.catalog]: configured and running, proceeding with master startup.
20:00:50.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.504902 24769 sys_catalog.cc:455] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:00:50.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.506673 24769 sys_catalog.cc:458] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [sys.catalog]: This master's current role is: FOLLOWER
20:00:50.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:50.557124 24774 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:00:50.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.557655 24757 raft_consensus.cc:493] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:00:50.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.568125 24757 raft_consensus.cc:515] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:00:50.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:50.581543 24482 tablet.cc:2406] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:00:50.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.588980 24714 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2882c6717446405899962890e037a3db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "426096099be0496fbc843c8b48c09fa4" is_pre_election: true
20:00:50.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.590444 24714 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2882c6717446405899962890e037a3db in term 0.
20:00:50.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.597072 24478 leader_election.cc:304] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2882c6717446405899962890e037a3db, 426096099be0496fbc843c8b48c09fa4; no voters:
20:00:50.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:50.596304 24774 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:00:50.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.587266 24757 leader_election.cc:290] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 14182101312f479992e2922ed266d990 (127.23.215.253:36309), 426096099be0496fbc843c8b48c09fa4 (127.23.215.252:40057)
20:00:50.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.601125 24757 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 0 FOLLOWER]: Leader pre-election won for term 1
20:00:50.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.601538 24757 raft_consensus.cc:493] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:00:50.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.602037 24757 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 0 FOLLOWER]: Advancing to term 1
20:00:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.529824 24450 master_runner.cc:187] Error getting master registration for 127.23.215.253:36309: OK, instance_id { permanent_uuid: "14182101312f479992e2922ed266d990" instance_seqno: 1786564847431623 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:00:50.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.653059 24757 raft_consensus.cc:515] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:00:50.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.665318 24714 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2882c6717446405899962890e037a3db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "426096099be0496fbc843c8b48c09fa4"
20:00:50.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.666463 24714 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [term 0 FOLLOWER]: Advancing to term 1
20:00:50.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.668527 24757 leader_election.cc:290] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [CANDIDATE]: Term 1 election: Requested vote from peers 14182101312f479992e2922ed266d990 (127.23.215.253:36309), 426096099be0496fbc843c8b48c09fa4 (127.23.215.252:40057)
20:00:50.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.669659 24450 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:00:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.694563 24714 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2882c6717446405899962890e037a3db in term 1.
20:00:50.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.696955 24478 leader_election.cc:304] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2882c6717446405899962890e037a3db, 426096099be0496fbc843c8b48c09fa4; no voters:
20:00:50.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.698297 24757 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 1 FOLLOWER]: Leader election won for term 1
20:00:50.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.699457 24757 raft_consensus.cc:697] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 1 LEADER]: Becoming Leader. State: Replica: 2882c6717446405899962890e037a3db, State: Running, Role: LEADER
20:00:50.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.701421 24757 consensus_queue.cc:237] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:00:50.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.718705 24778 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2882c6717446405899962890e037a3db. Latest consensus state: current_term: 1 leader_uuid: "2882c6717446405899962890e037a3db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:00:50.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.720249 24778 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [sys.catalog]: This master's current role is: LEADER
20:00:50.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:50.721187 24691 tablet.cc:2406] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:00:50.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.742606 24790 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:00:50.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.755122 24791 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
20:00:50.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.767604 24791 catalog_manager.cc:1520] Initializing Kudu cluster ID...
20:00:50.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.800110 24714 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [term 1 FOLLOWER]: Refusing update from remote peer 2882c6717446405899962890e037a3db: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:00:50.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.802114 24778 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [LEADER]: Connected to new peer: Peer: permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:00:50.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.786623 24649 master_runner.cc:187] Error getting master registration for 127.23.215.253:36309: OK, instance_id { permanent_uuid: "14182101312f479992e2922ed266d990" instance_seqno: 1786564847431623 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:00:50.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.837183 24649 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:00:50.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.892799 24757 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2882c6717446405899962890e037a3db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:00:50.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.893669 24757 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [sys.catalog]: This master's current role is: LEADER
20:00:50.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.896819 24791 catalog_manager.cc:1383] Generated new cluster ID: c813f4ceca5b42d7a69c90bdd53b0297
20:00:50.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.897246 24791 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
20:00:50.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.897143 24769 sys_catalog.cc:455] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2882c6717446405899962890e037a3db. Latest consensus state: current_term: 1 leader_uuid: "2882c6717446405899962890e037a3db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:00:50.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.898236 24769 sys_catalog.cc:458] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [sys.catalog]: This master's current role is: FOLLOWER
20:00:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.929162 24769 sys_catalog.cc:455] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2882c6717446405899962890e037a3db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:00:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.929877 24769 sys_catalog.cc:458] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [sys.catalog]: This master's current role is: FOLLOWER
20:00:50.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.984277 24791 catalog_manager.cc:1406] Generated new certificate authority record
20:00:50.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:50.986682 24791 catalog_manager.cc:1540] Loading token signing keys...
20:00:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:51.086551 24791 catalog_manager.cc:6092] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db: Generated new TSK 0
20:00:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:51.088088 24791 catalog_manager.cc:1550] Initializing in-progress tserver states...
20:00:51.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.23.215.193@KRBTEST.COM; defaulting to no policy
20:00:51.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:51.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.23.215.193@KRBTEST.COM" created.
20:00:51.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:51.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.215.193 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:51.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.215.193 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:51.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.23.215.193@KRBTEST.COM; defaulting to no policy
20:00:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.23.215.193@KRBTEST.COM" created.
20:00:51.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:51.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.215.193 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:51.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.215.193 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:51.347389 24415 mini_kdc.cc:268] Time spent creating service keytab for HTTP/127.23.215.193: real 0.131s user 0.000s sys 0.018s
20:00:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:51.350356 24415 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-0/wal
20:00:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-0/data
20:00:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-0/logs
20:00:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-0/data/info.pb
20:00:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.215.193:0
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.215.193
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.215.193
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.215.193
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.215.212:40571
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:51.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:51.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:00:51.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:51.765992 24790 catalog_manager.cc:1295] Loaded cluster ID: c813f4ceca5b42d7a69c90bdd53b0297
20:00:51.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:51.768378 24790 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4: loading cluster ID for follower catalog manager: success
20:00:51.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:51.857254 24790 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4: acquiring CA information for follower catalog manager: success
20:00:51.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:51.913064 24790 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:00:51.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:51.981114 24637 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } attempt: 8
20:00:52.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:52 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564847, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.253@KRBTEST.COM for kudu/127.23.215.252@KRBTEST.COM
20:00:52.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:52.239800 24480 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.23.215.253:36309 timed out after 1.662s (SENT) [suppressed 12 similar messages]
20:00:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.297837 24637 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990: Bootstrap starting.
20:00:52.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.325716 24637 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990: Neither blocks nor log segments found. Creating new log.
20:00:52.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.334669 24637 log.cc:826] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990: Log is configured to *not* fsync() on all Append() calls
20:00:52.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.360275 24637 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990: No bootstrap required, opened a new log
20:00:52.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.439596 24637 raft_consensus.cc:359] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:00:52.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.441402 24637 raft_consensus.cc:385] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:00:52.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.442117 24637 raft_consensus.cc:740] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 14182101312f479992e2922ed266d990, State: Initialized, Role: FOLLOWER
20:00:52.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.444540 24637 consensus_queue.cc:260] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:00:52.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.472155 24817 sys_catalog.cc:455] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:00:52.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.473414 24817 sys_catalog.cc:458] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [sys.catalog]: This master's current role is: FOLLOWER
20:00:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.474048 24817 raft_consensus.cc:493] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:00:52.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.474794 24817 raft_consensus.cc:515] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:00:52.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.482540 24606 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2882c6717446405899962890e037a3db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "14182101312f479992e2922ed266d990" is_pre_election: true
20:00:52.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:52.483959 24606 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:56816 (request call id 2) took 1905 ms (client timeout 1662 ms). Trace:
20:00:52.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:52.484580 24606 rpcz_store.cc:269] 0812 20:00:50.578601 (+ 0us) service_pool.cc:167] Inserting onto call queue
20:00:52.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:00:50.578825 (+ 224us) service_pool.cc:224] Handling call
20:00:52.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:00:52.483836 (+1905011us) inbound_call.cc:177] Queueing success response
20:00:52.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
20:00:52.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.486676 24603 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2882c6717446405899962890e037a3db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "14182101312f479992e2922ed266d990" is_pre_election: true
20:00:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.478066 24637 sys_catalog.cc:565] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [sys.catalog]: configured and running, proceeding with master startup.
20:00:52.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.491633 24604 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [term 0 FOLLOWER]: Advancing to term 1
20:00:52.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.494326 24605 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2882c6717446405899962890e037a3db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "14182101312f479992e2922ed266d990"
20:00:52.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:52.496523 24605 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:56816 (request call id 3) took 1835 ms (client timeout 1792 ms). Trace:
20:00:52.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:52.497053 24605 rpcz_store.cc:269] 0812 20:00:50.661477 (+ 0us) service_pool.cc:167] Inserting onto call queue
20:00:52.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:00:50.661695 (+ 218us) service_pool.cc:224] Handling call
20:00:52.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:00:52.496411 (+1834716us) inbound_call.cc:177] Queueing success response
20:00:52.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
20:00:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.499985 24714 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "14182101312f479992e2922ed266d990" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "426096099be0496fbc843c8b48c09fa4" is_pre_election: true
20:00:52.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.494709 24602 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2882c6717446405899962890e037a3db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "14182101312f479992e2922ed266d990"
20:00:52.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.502938 24817 leader_election.cc:290] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2882c6717446405899962890e037a3db (127.23.215.254:39917), 426096099be0496fbc843c8b48c09fa4 (127.23.215.252:40057)
20:00:52.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.507031 24509 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "14182101312f479992e2922ed266d990" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2882c6717446405899962890e037a3db" is_pre_election: true
20:00:52.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.512570 24576 leader_election.cc:304] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 14182101312f479992e2922ed266d990; no voters: 2882c6717446405899962890e037a3db, 426096099be0496fbc843c8b48c09fa4
20:00:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.523298 24604 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [term 1 FOLLOWER]: Refusing update from remote peer 2882c6717446405899962890e037a3db: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:00:52.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.529374 24604 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:56816 (request call id 4) took 1723 ms. Trace:
20:00:52.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.529644 24818 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:00:52.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.529892 24604 rpcz_store.cc:276] 0812 20:00:50.806273 (+ 0us) service_pool.cc:167] Inserting onto call queue
20:00:52.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:00:50.806446 (+ 173us) service_pool.cc:224] Handling call
20:00:52.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:00:52.489482 (+1683036us) raft_consensus.cc:1471] Updating replica for 0 ops
20:00:52.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:00:52.528807 (+ 39325us) raft_consensus.cc:1697] Filling consensus response to leader.
20:00:52.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:00:52.529260 (+ 453us) inbound_call.cc:177] Queueing success response
20:00:52.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
20:00:52.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.536618 24807 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [LEADER]: Connected to new peer: Peer: permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:00:52.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:52.618147 24802 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:00:52.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:52.619397 24802 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:52.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:52.620642 24802 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:00:52.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:52.638383 24581 tablet.cc:2406] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:00:52.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.669246 24822 mvcc.cc:204] Tried to move back new op lower bound from 7317769628842729472 to 7317769628492173312. Current Snapshot: MvccSnapshot[applied={T|T < 7317769628842729472}]
20:00:52.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.740564 24818 sys_catalog.cc:455] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2882c6717446405899962890e037a3db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:00:52.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.743268 24817 sys_catalog.cc:455] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2882c6717446405899962890e037a3db. Latest consensus state: current_term: 1 leader_uuid: "2882c6717446405899962890e037a3db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:00:52.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.744620 24817 sys_catalog.cc:458] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [sys.catalog]: This master's current role is: FOLLOWER
20:00:52.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.746255 24818 sys_catalog.cc:458] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [sys.catalog]: This master's current role is: FOLLOWER
20:00:52.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:52.754262 24802 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:00:52.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:52.755930 24802 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.215.193
20:00:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.841218 24838 catalog_manager.cc:1295] Loaded cluster ID: c813f4ceca5b42d7a69c90bdd53b0297
20:00:52.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.842124 24838 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990: loading cluster ID for follower catalog manager: success
20:00:52.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.862443 24838 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990: acquiring CA information for follower catalog manager: success
20:00:52.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.870741 24838 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:00:52.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.891636 24802 tablet_server_runner.cc:78] Tablet server non-default flags:
20:00:52.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-0/logs
20:00:52.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:52.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:52.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:52.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.215.212:40571
20:00:52.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:52.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-0/data
20:00:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-0/wal
20:00:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab
20:00:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.215.193
20:00:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.215.193:0
20:00:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:00:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:00:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:00:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:52.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-0/data/info.pb
20:00:52.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.215.193
20:00:52.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:52.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057
20:00:52.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:52.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:52.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:52.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:52.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:52.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.215.193
20:00:52.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:52.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:00:52.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:00:52.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 565fa2a0c8b8655dfe08c1dca95e4b4d9408fc9c
20:00:52.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:52.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 19:45:48 UTC on e7f111948823
20:00:52.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13311
20:00:52.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:52.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.893543 24802 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:52.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:52.895268 24802 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:00:52.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:52.963189 24841 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:53.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:53.098801 24807 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2882c6717446405899962890e037a3db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:00:53.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:53.099733 24807 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [sys.catalog]: This master's current role is: LEADER
20:00:54.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:54 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564854, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:00:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:54.724763 24470 init.cc:295] Successfully reacquired a new kerberos TGT
20:00:55.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:54.326848 24840 debug-util.cc:398] Leaking SignalData structure 0x7b0800028380 after lost signal to thread 24802
20:00:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:55.858868 24843 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2874 milliseconds
20:00:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:55.859578 24802 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.876s user 0.002s sys 0.002s
20:00:55.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:55.861634 24802 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.878s user 0.002s sys 0.002s
20:00:55.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:52.979571 24842 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:55.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:55.867612 24802 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:00:55.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:55.868935 24845 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:55.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:55.870625 24802 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:55.881662 24802 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:55.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:55.885537 24802 hybrid_clock.cc:648] HybridClock initialized: now 1786564855885488 us; error 55 us; skew 500 ppm
20:00:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:55.886826 24802 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:55.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:55 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564855, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:00:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:55.896649 24802 init.cc:377] Logged in from keytab as kudu/127.23.215.193@KRBTEST.COM (short username kudu)
20:00:55.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:55.942229 24802 webserver.cc:533] Webserver started at http://127.23.215.193:35915/ using document root <none> and password file <none>
20:00:55.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:55.944180 24802 fs_manager.cc:362] Metadata directory not provided
20:00:55.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:55.944877 24802 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:55.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:55.945694 24802 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:00:55.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:55.961784 24802 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-0/data/instance:
20:00:55.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6da5284a5f934a50ab8431e9ee94145f"
20:00:55.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:55 on dist-test-slave-1jjb"
20:00:55.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:55.963863 24802 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-0/wal/instance:
20:00:55.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6da5284a5f934a50ab8431e9ee94145f"
20:00:55.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:55 on dist-test-slave-1jjb"
20:00:56.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:56.000921 24802 fs_manager.cc:696] Time spent creating directory manager: real 0.036s user 0.008s sys 0.005s
20:00:56.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:56.014837 24861 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:56.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:56.016744 24802 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.003s sys 0.003s
20:00:56.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:56.017556 24802 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-0/data,/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-0/wal
20:00:56.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6da5284a5f934a50ab8431e9ee94145f"
20:00:56.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:55 on dist-test-slave-1jjb"
20:00:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:56.018258 24802 fs_report.cc:389] FS layout report
20:00:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-0/wal
20:00:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-0/wal
20:00:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-0/data/data
20:00:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:56.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:56.198575 24802 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:00:56.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:56.226388 24802 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:56.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:56.227716 24802 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:56.258958 24802 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:00:56.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:56.266062 24802 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:00:56.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:56.266359 24802 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:00:56.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:56.266639 24802 ts_tablet_manager.cc:616] Registered 0 tablets
20:00:56.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:56.266861 24802 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:00:57.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:57 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564857, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:00:57.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.288235 24565 init.cc:295] Successfully reacquired a new kerberos TGT
20:00:57.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.560607 24802 rpc_server.cc:307] RPC server started. Bound to: 127.23.215.193:34191
20:00:57.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.561213 24995 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.215.193:34191 every 8 connection(s)
20:00:57.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.564904 24802 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-0/data/info.pb
20:00:57.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.573863 24415 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu as pid 24802
20:00:57.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.574452 24415 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-0/wal/instance
20:00:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.23.215.194@KRBTEST.COM; defaulting to no policy
20:00:57.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:57.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.23.215.194@KRBTEST.COM" created.
20:00:57.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:57 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564855, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.193@KRBTEST.COM for kudu/127.23.215.254@KRBTEST.COM
20:00:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:57 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564855, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.193@KRBTEST.COM for kudu/127.23.215.253@KRBTEST.COM
20:00:57.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:57 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564855, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.193@KRBTEST.COM for kudu/127.23.215.252@KRBTEST.COM
20:00:57.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.729190 24998 heartbeater.cc:344] Connected to a master server at 127.23.215.254:39917
20:00:57.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.730028 24998 heartbeater.cc:461] Registering TS with master...
20:00:57.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.733429 24998 heartbeater.cc:507] Master 127.23.215.254:39917 requested a full tablet report, sending...
20:00:57.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.739432 24498 ts_manager.cc:194] Registered new tserver with Master: 6da5284a5f934a50ab8431e9ee94145f (127.23.215.193:34191)
20:00:57.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.744423 24498 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.23.215.193@KRBTEST.COM'} at 127.23.215.193:49947
20:00:57.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.746274 24997 heartbeater.cc:344] Connected to a master server at 127.23.215.253:36309
20:00:57.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.747076 24997 heartbeater.cc:461] Registering TS with master...
20:00:57.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.749915 24997 heartbeater.cc:507] Master 127.23.215.253:36309 requested a full tablet report, sending...
20:00:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.755927 24593 ts_manager.cc:194] Registered new tserver with Master: 6da5284a5f934a50ab8431e9ee94145f (127.23.215.193:34191)
20:00:57.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.760625 24996 heartbeater.cc:344] Connected to a master server at 127.23.215.252:40057
20:00:57.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.761173 24996 heartbeater.cc:461] Registering TS with master...
20:00:57.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.762351 24996 heartbeater.cc:507] Master 127.23.215.252:40057 requested a full tablet report, sending...
20:00:57.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.772567 24703 ts_manager.cc:194] Registered new tserver with Master: 6da5284a5f934a50ab8431e9ee94145f (127.23.215.193:34191)
20:00:57.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:57.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.215.194 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:57.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.215.194 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:57.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:57.775888 24415 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.23.215.194: real 0.178s user 0.000s sys 0.044s
20:00:57.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.23.215.194@KRBTEST.COM; defaulting to no policy
20:00:57.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:57.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.23.215.194@KRBTEST.COM" created.
20:00:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:00:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.215.194 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.215.194 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:00:57.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:57.866834 24415 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:57.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:00:57.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-1/wal
20:00:57.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-1/data
20:00:57.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:57.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-1/logs
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-1/data/info.pb
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.215.194:0
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.215.194
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.215.194
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.215.194
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.215.212:40571
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:57.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:00:58.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:58.365494 25012 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:00:58.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:58.366174 25012 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:58.367165 25012 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:00:58.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:58.422227 25012 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:00:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:58.423148 25012 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.215.194
20:00:58.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:58.467433 25012 tablet_server_runner.cc:78] Tablet server non-default flags:
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-1/logs
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.215.212:40571
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-1/data
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-1/wal
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.215.194
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.215.194:0
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-1/data/info.pb
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.215.194
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:58.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:58.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:58.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:58.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.215.194
20:00:58.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:58.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:00:58.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:00:58.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 565fa2a0c8b8655dfe08c1dca95e4b4d9408fc9c
20:00:58.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:58.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 19:45:48 UTC on e7f111948823
20:00:58.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13311
20:00:58.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:58.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:58.469266 25012 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:58.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:58.470724 25012 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:00:58.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:58.499294 25023 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:58.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:58.753439 24998 heartbeater.cc:499] Master 127.23.215.254:39917 was elected leader, sending a full tablet report...
20:00:59.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:00:59 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564859, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:00:59.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:00:59.879156 24674 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:00.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:00.378044 25012 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.864s user 0.519s sys 1.072s
20:01:00.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:59.897856 25020 debug-util.cc:398] Leaking SignalData structure 0x7b0800028380 after lost signal to thread 25012
20:01:00.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:00.379236 25012 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.865s user 0.519s sys 1.072s
20:01:00.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:00:58.516438 25024 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:00.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:00.385195 25026 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:00.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.390288 25012 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:00.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:00.390218 25025 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1862 milliseconds
20:01:00.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.391808 25012 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:00.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.396922 25012 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.398396 25012 hybrid_clock.cc:648] HybridClock initialized: now 1786564860398301 us; error 110 us; skew 500 ppm
20:01:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.399611 25012 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:00.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:00 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564860, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:00.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.410482 25012 init.cc:377] Logged in from keytab as kudu/127.23.215.194@KRBTEST.COM (short username kudu)
20:01:00.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.424253 25012 webserver.cc:533] Webserver started at http://127.23.215.194:37327/ using document root <none> and password file <none>
20:01:00.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.425391 25012 fs_manager.cc:362] Metadata directory not provided
20:01:00.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.425647 25012 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:00.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.426245 25012 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:01:00.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.432219 25012 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-1/data/instance:
20:01:00.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26cee1d4b3434035a2d70d2ee68c989a"
20:01:00.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:01:00 on dist-test-slave-1jjb"
20:01:00.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.433463 25012 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-1/wal/instance:
20:01:00.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26cee1d4b3434035a2d70d2ee68c989a"
20:01:00.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:01:00 on dist-test-slave-1jjb"
20:01:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.446288 25012 fs_manager.cc:696] Time spent creating directory manager: real 0.012s user 0.008s sys 0.004s
20:01:00.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.457789 25038 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:00.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.459801 25012 fs_manager.cc:730] Time spent opening block manager: real 0.010s user 0.002s sys 0.007s
20:01:00.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.460376 25012 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-1/data,/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-1/wal
20:01:00.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26cee1d4b3434035a2d70d2ee68c989a"
20:01:00.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:01:00 on dist-test-slave-1jjb"
20:01:00.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.460924 25012 fs_report.cc:389] FS layout report
20:01:00.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-1/wal
20:01:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-1/wal
20:01:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-1/data/data
20:01:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.545044 25012 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:00.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.553543 25012 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:00.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.554077 25012 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:00.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.560575 25012 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:01:00.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.570290 25012 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:01:00.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.570555 25012 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:01:00.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.570919 25012 ts_tablet_manager.cc:616] Registered 0 tablets
20:01:00.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:00.571134 25012 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:01:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.206382 25012 rpc_server.cc:307] RPC server started. Bound to: 127.23.215.194:40345
20:01:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.206553 25171 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.215.194:40345 every 8 connection(s)
20:01:01.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.209702 25012 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-1/data/info.pb
20:01:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.221969 24415 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu as pid 25012
20:01:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.223110 24415 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-1/wal/instance
20:01:01.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:01 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564860, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.194@KRBTEST.COM for kudu/127.23.215.252@KRBTEST.COM
20:01:01.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.23.215.195@KRBTEST.COM; defaulting to no policy
20:01:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:01 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564860, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.194@KRBTEST.COM for kudu/127.23.215.253@KRBTEST.COM
20:01:01.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:01.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.23.215.195@KRBTEST.COM" created.
20:01:01.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:01 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564860, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.194@KRBTEST.COM for kudu/127.23.215.254@KRBTEST.COM
20:01:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.345641 25172 heartbeater.cc:344] Connected to a master server at 127.23.215.252:40057
20:01:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.346379 25172 heartbeater.cc:461] Registering TS with master...
20:01:01.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.348845 25172 heartbeater.cc:507] Master 127.23.215.252:40057 requested a full tablet report, sending...
20:01:01.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.354351 25173 heartbeater.cc:344] Connected to a master server at 127.23.215.253:36309
20:01:01.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.354717 25173 heartbeater.cc:461] Registering TS with master...
20:01:01.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.354414 24703 ts_manager.cc:194] Registered new tserver with Master: 26cee1d4b3434035a2d70d2ee68c989a (127.23.215.194:40345)
20:01:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.355746 25173 heartbeater.cc:507] Master 127.23.215.253:36309 requested a full tablet report, sending...
20:01:01.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.360338 24593 ts_manager.cc:194] Registered new tserver with Master: 26cee1d4b3434035a2d70d2ee68c989a (127.23.215.194:40345)
20:01:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.363693 25178 heartbeater.cc:344] Connected to a master server at 127.23.215.254:39917
20:01:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.364267 25178 heartbeater.cc:461] Registering TS with master...
20:01:01.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.366232 25178 heartbeater.cc:507] Master 127.23.215.254:39917 requested a full tablet report, sending...
20:01:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.369467 24498 ts_manager.cc:194] Registered new tserver with Master: 26cee1d4b3434035a2d70d2ee68c989a (127.23.215.194:40345)
20:01:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.371510 24498 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.23.215.194@KRBTEST.COM'} at 127.23.215.194:52571
20:01:01.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:01.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.215.195 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:01:01.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.215.195 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:01:01.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:01.381330 24415 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.23.215.195: real 0.143s user 0.001s sys 0.040s
20:01:01.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.23.215.195@KRBTEST.COM; defaulting to no policy
20:01:01.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:01.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.23.215.195@KRBTEST.COM" created.
20:01:01.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:01.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.215.195 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:01:01.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.215.195 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab.
20:01:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:01.441386 24415 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:01:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:01:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-2/wal
20:01:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-2/data
20:01:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-2/logs
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-2/data/info.pb
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.215.195:0
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.215.195
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.215.195
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.215.195
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.215.212:40571
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:01.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:01.922974 25191 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:01.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:01.924116 25191 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:01.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:01.924973 25191 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:01.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:01.962221 25191 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:01.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:01.963132 25191 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.215.195
20:01:02.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:02.002658 25191 tablet_server_runner.cc:78] Tablet server non-default flags:
20:01:02.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-2/logs
20:01:02.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:02.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.215.212:40571
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-2/data
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-2/wal
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.215.195
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.215.195:0
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-2/data/info.pb
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.215.195
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:02.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.215.195
20:01:02.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:02.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:01:02.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:02.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 565fa2a0c8b8655dfe08c1dca95e4b4d9408fc9c
20:01:02.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:02.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 19:45:48 UTC on e7f111948823
20:01:02.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13311
20:01:02.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:02.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:02.004369 25191 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:02.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:02.005746 25191 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:02.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:02.023738 25199 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:02.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:02.375604 25178 heartbeater.cc:499] Master 127.23.215.254:39917 was elected leader, sending a full tablet report...
20:01:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:03.426838 25198 debug-util.cc:398] Leaking SignalData structure 0x7b0800028380 after lost signal to thread 25191
20:01:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:03.946205 25198 kernel_stack_watchdog.cc:198] Thread 25191 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 396ms:
20:01:03.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:01:03.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:01:03.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:01:03.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:01:03.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:02.026762 25200 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:03.950790 25202 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1920 milliseconds
20:01:03.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:03.956718 25191 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.929s user 0.000s sys 0.004s
20:01:03.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:03.957060 25191 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.929s user 0.000s sys 0.004s
20:01:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:03.958423 25203 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:03.958442 25191 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:03.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:03.959909 25191 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:03.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:03.963629 25191 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:03.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:03.965101 25191 hybrid_clock.cc:648] HybridClock initialized: now 1786564863965056 us; error 45 us; skew 500 ppm
20:01:03.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:03.965975 25191 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:03.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:03 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564863, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:03.974788 25191 init.cc:377] Logged in from keytab as kudu/127.23.215.195@KRBTEST.COM (short username kudu)
20:01:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:03.983907 25191 webserver.cc:533] Webserver started at http://127.23.215.195:35241/ using document root <none> and password file <none>
20:01:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:03.985158 25191 fs_manager.cc:362] Metadata directory not provided
20:01:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:03.985507 25191 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:03.986093 25191 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:01:03.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:03.993310 25191 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-2/data/instance:
20:01:03.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ba524228f99842d688a1b28b407d61bc"
20:01:03.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:01:03 on dist-test-slave-1jjb"
20:01:03.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:03.994944 25191 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-2/wal/instance:
20:01:03.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ba524228f99842d688a1b28b407d61bc"
20:01:03.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:01:03 on dist-test-slave-1jjb"
20:01:04.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.007848 25191 fs_manager.cc:696] Time spent creating directory manager: real 0.012s user 0.005s sys 0.009s
20:01:04.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.016062 25219 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:04.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.017592 25191 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.006s sys 0.002s
20:01:04.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.017887 25191 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-2/data,/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-2/wal
20:01:04.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ba524228f99842d688a1b28b407d61bc"
20:01:04.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:01:03 on dist-test-slave-1jjb"
20:01:04.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.018286 25191 fs_report.cc:389] FS layout report
20:01:04.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:04.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-2/wal
20:01:04.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-2/wal
20:01:04.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-2/data/data
20:01:04.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:04.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.083935 25191 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:04.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.091699 25191 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:04.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.092294 25191 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.096413 25191 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:01:04.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.102097 25191 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:01:04.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.102371 25191 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:01:04.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.102658 25191 ts_tablet_manager.cc:616] Registered 0 tablets
20:01:04.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.102854 25191 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:01:04.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.652217 25191 rpc_server.cc:307] RPC server started. Bound to: 127.23.215.195:36545
20:01:04.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.652630 25355 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.215.195:36545 every 8 connection(s)
20:01:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.656286 25191 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-2/data/info.pb
20:01:04.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.659842 24415 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu as pid 25191
20:01:04.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.661336 24415 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/ts-2/wal/instance
20:01:04.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:04 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564864, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:04.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.731397 24470 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:04.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:04 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564863, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.195@KRBTEST.COM for kudu/127.23.215.252@KRBTEST.COM
20:01:04.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:04 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564863, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.195@KRBTEST.COM for kudu/127.23.215.253@KRBTEST.COM
20:01:04.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:04 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564863, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.195@KRBTEST.COM for kudu/127.23.215.254@KRBTEST.COM
20:01:04.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.773452 25356 heartbeater.cc:344] Connected to a master server at 127.23.215.252:40057
20:01:04.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.773983 25356 heartbeater.cc:461] Registering TS with master...
20:01:04.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.776031 25356 heartbeater.cc:507] Master 127.23.215.252:40057 requested a full tablet report, sending...
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.781213 24703 ts_manager.cc:194] Registered new tserver with Master: ba524228f99842d688a1b28b407d61bc (127.23.215.195:36545)
20:01:04.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.783596 25357 heartbeater.cc:344] Connected to a master server at 127.23.215.253:36309
20:01:04.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.784049 25357 heartbeater.cc:461] Registering TS with master...
20:01:04.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.785828 25357 heartbeater.cc:507] Master 127.23.215.253:36309 requested a full tablet report, sending...
20:01:04.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.790223 24593 ts_manager.cc:194] Registered new tserver with Master: ba524228f99842d688a1b28b407d61bc (127.23.215.195:36545)
20:01:04.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.795244 25361 heartbeater.cc:344] Connected to a master server at 127.23.215.254:39917
20:01:04.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.795701 25361 heartbeater.cc:461] Registering TS with master...
20:01:04.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.797566 25361 heartbeater.cc:507] Master 127.23.215.254:39917 requested a full tablet report, sending...
20:01:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.802088 24498 ts_manager.cc:194] Registered new tserver with Master: ba524228f99842d688a1b28b407d61bc (127.23.215.195:36545)
20:01:04.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.805091 24498 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.23.215.195@KRBTEST.COM'} at 127.23.215.195:45987
20:01:04.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:04.808050 24415 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
20:01:04.845 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:04.846 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
20:01:04.856 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
20:01:04.857 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
20:01:04.860 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.23.215.254"
port: 39917
}
bound_http_address {
host: "127.23.215.254"
port: 46357
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.23.215.253"
port: 36309
}
bound_http_address {
host: "127.23.215.253"
port: 37699
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.23.215.252"
port: 40057
}
bound_http_address {
host: "127.23.215.252"
port: 34117
}
}
}
20:01:04.861 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:01:04.862 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.23.215.193"
port: 34191
}
bound_http_address {
host: "127.23.215.193"
port: 35915
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.23.215.194"
port: 40345
}
bound_http_address {
host: "127.23.215.194"
port: 37327
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.23.215.195"
port: 36545
}
bound_http_address {
host: "127.23.215.195"
port: 35241
}
}
}
20:01:04.862 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:01:04.863 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:01:04.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:04 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:04.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:01:04.920 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:04.923 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5cc
20:01:04.937 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:01:04.937 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5cc
20:01:04.939 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:01:04.965 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
20:01:04.968 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:01:04.968 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:01:05.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:05 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564865, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:05.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:01:05.013 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:05.013 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5cc
20:01:05.014 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:01:05.015 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.23.215.254:39917
20:01:05.016 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:01:05.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:05.017323 24415 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu with pid 24450
20:01:05.051 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:05.052 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.23.215.253:36309
20:01:05.052 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:01:05.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:05.053047 24415 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu with pid 24548
20:01:05.100 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:05.100 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.23.215.252:40057
20:01:05.100 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:01:05.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:05.101161 24415 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu with pid 24649
20:01:05.131 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:05.131 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.23.215.254:39917
20:01:05.135 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:01:05.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:05.137275 24415 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:01:05.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:01:05.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/wal
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/data
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/logs
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/data/info.pb
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.215.254:39917
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.215.254
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46357
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.215.254
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.215.212:40571
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34355/jwks.json
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:01:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:05.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:05.394905 25172 heartbeater.cc:646] Failed to heartbeat to 127.23.215.252:40057 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:01:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:05.467772 25376 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:05.468452 25376 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:05.469133 25376 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:05.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:05.508476 25376 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:05.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:05.508697 25376 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:01:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:05.508977 25376 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:05.509198 25376 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:05.509402 25376 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:01:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:05.509600 25376 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:05.509753 25376 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34355/jwks.json
20:01:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:05.549383 25376 master_runner.cc:387] Master server non-default flags:
20:01:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/logs
20:01:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.215.212:40571
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/data
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/wal
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.215.254
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.215.254:39917
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34355/jwks.json
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/data/info.pb
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.215.254
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46357
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 565fa2a0c8b8655dfe08c1dca95e4b4d9408fc9c
20:01:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 19:45:48 UTC on e7f111948823
20:01:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13311
20:01:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:05.550961 25376 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:05.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:05.552402 25376 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:05.565719 25383 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:05.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:05.799822 25356 heartbeater.cc:646] Failed to heartbeat to 127.23.215.252:40057 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:01:05.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:05.811153 25361 heartbeater.cc:499] Master 127.23.215.254:39917 was elected leader, sending a full tablet report...
20:01:05.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:05.853228 24997 heartbeater.cc:646] Failed to heartbeat to 127.23.215.253:36309 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.23.215.253:36309: connect: Connection refused (error 111)
20:01:05.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:05 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564865, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:05.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:05.915933 24857 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:06.970454 25382 debug-util.cc:398] Leaking SignalData structure 0x7b0800034fe0 after lost signal to thread 25376
20:01:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:05.566898 25384 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:07.298208 25382 kernel_stack_watchdog.cc:198] Thread 25376 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
20:01:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:01:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:01:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:01:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:01:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:07.299381 25376 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.733s user 0.591s sys 1.051s
20:01:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:07.299747 25376 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.734s user 0.591s sys 1.051s
20:01:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:07.300441 25386 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:07.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.304939 25376 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:07.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:07.306527 25385 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1735 milliseconds
20:01:07.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.309391 25376 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:07.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.314319 25376 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:07.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.315796 25376 hybrid_clock.cc:648] HybridClock initialized: now 1786564867315745 us; error 60 us; skew 500 ppm
20:01:07.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.316834 25376 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:07 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564867, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:07.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.323755 25376 init.cc:377] Logged in from keytab as kudu/127.23.215.254@KRBTEST.COM (short username kudu)
20:01:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.329977 25376 webserver.cc:533] Webserver started at http://127.23.215.254:46357/ using document root <none> and password file <none>
20:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.330875 25376 fs_manager.cc:362] Metadata directory not provided
20:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.331094 25376 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:07.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.339756 25376 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
20:01:07.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.344501 25413 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:07.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.345876 25376 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
20:01:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.346221 25376 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/data,/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/wal
20:01:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2882c6717446405899962890e037a3db"
20:01:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:44 on dist-test-slave-1jjb"
20:01:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.348222 25376 fs_report.cc:389] FS layout report
20:01:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/wal
20:01:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/wal
20:01:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/data/data
20:01:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:07.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.411250 25376 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.416916 25376 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.417346 25376 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.501849 25376 rpc_server.cc:307] RPC server started. Bound to: 127.23.215.254:39917
20:01:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.501926 25482 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.215.254:39917 every 8 connection(s)
20:01:07.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.504628 25376 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-0/data/info.pb
20:01:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.508627 24415 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu as pid 25376
20:01:07.509 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:07.509 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.23.215.253:36309
20:01:07.509 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:01:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.510946 24415 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:01:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:01:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/wal
20:01:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/data
20:01:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/logs
20:01:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/data/info.pb
20:01:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.215.253:36309
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.215.253
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37699
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.215.253
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:07.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:07.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.215.212:40571
20:01:07.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:07.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:07.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:07.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:07.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057
20:01:07.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:07.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34355/jwks.json
20:01:07.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:01:07.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:07.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.517869 25483 sys_catalog.cc:263] Verifying existing consensus state
20:01:07.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.525725 25483 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db: Bootstrap starting.
20:01:07.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.569687 25483 log.cc:826] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db: Log is configured to *not* fsync() on all Append() calls
20:01:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.585915 25483 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:01:07.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.586990 25483 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db: Bootstrap complete.
20:01:07.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.615888 25483 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:01:07.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.618458 25483 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2882c6717446405899962890e037a3db, State: Initialized, Role: FOLLOWER
20:01:07.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.619537 25483 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:01:07.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.622473 25488 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:01:07.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.623423 25488 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [sys.catalog]: This master's current role is: FOLLOWER
20:01:07.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.623664 25483 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [sys.catalog]: configured and running, proceeding with master startup.
20:01:07.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.669624 25501 catalog_manager.cc:1295] Loaded cluster ID: c813f4ceca5b42d7a69c90bdd53b0297
20:01:07.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.669898 25501 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db: loading cluster ID for follower catalog manager: success
20:01:07.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.678905 25501 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db: acquiring CA information for follower catalog manager: success
20:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:07 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564867, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.254@KRBTEST.COM for kudu/127.23.215.254@KRBTEST.COM
20:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.684687 25501 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:01:07.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.704423 25376 master_runner.cc:187] Error getting master registration for 127.23.215.253:36309: Network error: Client connection negotiation failed: client connection to 127.23.215.253:36309: connect: Connection refused (error 111),
20:01:07.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.733878 25376 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:01:07.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.834695 25361 heartbeater.cc:344] Connected to a master server at 127.23.215.254:39917
20:01:07.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.835232 25361 heartbeater.cc:499] Master 127.23.215.254:39917 was elected leader, sending a full tablet report...
20:01:07.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.842470 25437 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ba524228f99842d688a1b28b407d61bc" instance_seqno: 1786564864518653) as {username='kudu', principal='kudu/127.23.215.195@KRBTEST.COM'} at 127.23.215.195:59757; Asking this server to re-register.
20:01:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.844659 25361 heartbeater.cc:461] Registering TS with master...
20:01:07.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.848117 25437 ts_manager.cc:194] Registered new tserver with Master: ba524228f99842d688a1b28b407d61bc (127.23.215.195:36545)
20:01:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:07.892881 25485 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:07.893571 25485 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:07.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:07.894356 25485 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.943632 24998 heartbeater.cc:344] Connected to a master server at 127.23.215.254:39917
20:01:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:07.944229 25485 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:07.944523 25485 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:01:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:07.945004 25485 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:07.945349 25485 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:07.945683 25485 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:01:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:07.946020 25485 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:07.946271 25485 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34355/jwks.json
20:01:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.946314 25437 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "6da5284a5f934a50ab8431e9ee94145f" instance_seqno: 1786564857268799) as {username='kudu', principal='kudu/127.23.215.193@KRBTEST.COM'} at 127.23.215.193:56267; Asking this server to re-register.
20:01:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.947970 24998 heartbeater.cc:461] Registering TS with master...
20:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.951718 25437 ts_manager.cc:194] Registered new tserver with Master: 6da5284a5f934a50ab8431e9ee94145f (127.23.215.193:34191)
20:01:07.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.994762 25485 master_runner.cc:387] Master server non-default flags:
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/logs
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.215.212:40571
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/data
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/wal
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.215.253
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.215.253:36309
20:01:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34355/jwks.json
20:01:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/data/info.pb
20:01:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.215.253
20:01:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37699
20:01:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 565fa2a0c8b8655dfe08c1dca95e4b4d9408fc9c
20:01:07.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:07.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 19:45:48 UTC on e7f111948823
20:01:07.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13311
20:01:07.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:07.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.996435 25485 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:07.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:07.997977 25485 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:08.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:08.012017 25518 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:08.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:08.484884 25178 heartbeater.cc:344] Connected to a master server at 127.23.215.254:39917
20:01:08.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:08.490051 25437 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "26cee1d4b3434035a2d70d2ee68c989a" instance_seqno: 1786564861055711) as {username='kudu', principal='kudu/127.23.215.194@KRBTEST.COM'} at 127.23.215.194:59405; Asking this server to re-register.
20:01:08.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:08.491724 25178 heartbeater.cc:461] Registering TS with master...
20:01:08.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:08.495721 25437 ts_manager.cc:194] Registered new tserver with Master: 26cee1d4b3434035a2d70d2ee68c989a (127.23.215.194:40345)
20:01:08.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:08.901535 25531 raft_consensus.cc:493] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:01:08.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:08.903838 25531 raft_consensus.cc:515] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:01:08.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:08.933596 25531 leader_election.cc:290] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 14182101312f479992e2922ed266d990 (127.23.215.253:36309), 426096099be0496fbc843c8b48c09fa4 (127.23.215.252:40057)
20:01:08.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:08.975016 25418 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.215.253:36309: connect: Connection refused (error 111)
20:01:08.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:08.985522 25418 leader_election.cc:336] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 14182101312f479992e2922ed266d990 (127.23.215.253:36309): Network error: Client connection negotiation failed: client connection to 127.23.215.253:36309: connect: Connection refused (error 111)
20:01:08.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:08.990600 25417 leader_election.cc:336] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 426096099be0496fbc843c8b48c09fa4 (127.23.215.252:40057): Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:01:08.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:08.993161 25417 leader_election.cc:304] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2882c6717446405899962890e037a3db; no voters: 14182101312f479992e2922ed266d990, 426096099be0496fbc843c8b48c09fa4
20:01:08.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:08.994217 25531 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
20:01:09.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:09.307390 25351 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba700 after lost signal to thread 25192
20:01:09.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:09.309108 25351 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5e00 after lost signal to thread 25355
20:01:10.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:09.417608 25517 debug-util.cc:398] Leaking SignalData structure 0x7b0800034fe0 after lost signal to thread 25485
20:01:10.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:10.010078 25517 kernel_stack_watchdog.cc:198] Thread 25485 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
20:01:10.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:01:10.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:01:10.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:01:10.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:01:10.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:08.013953 25519 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:10.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:10.015535 25485 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.004s user 0.674s sys 1.121s
20:01:10.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:10.015928 25485 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.004s user 0.674s sys 1.121s
20:01:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.020516 25485 server_base.cc:1061] running on GCE node
20:01:10.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:10.021380 25549 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:10.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.022687 25485 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.025804 25485 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:10.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.027356 25485 hybrid_clock.cc:648] HybridClock initialized: now 1786564870027313 us; error 42 us; skew 500 ppm
20:01:10.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.028308 25485 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:10.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:10 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564870, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:10.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.041085 25485 init.cc:377] Logged in from keytab as kudu/127.23.215.253@KRBTEST.COM (short username kudu)
20:01:10.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.089746 25485 webserver.cc:533] Webserver started at http://127.23.215.253:37699/ using document root <none> and password file <none>
20:01:10.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.091238 25485 fs_manager.cc:362] Metadata directory not provided
20:01:10.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.091588 25485 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.113936 25485 fs_manager.cc:714] Time spent opening directory manager: real 0.017s user 0.006s sys 0.010s
20:01:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.121678 25555 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.123153 25485 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
20:01:10.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.123589 25485 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/data,/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/wal
20:01:10.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "14182101312f479992e2922ed266d990"
20:01:10.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:47 on dist-test-slave-1jjb"
20:01:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.126411 25485 fs_report.cc:389] FS layout report
20:01:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/wal
20:01:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/wal
20:01:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/data/data
20:01:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:10.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:10.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:10.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:10.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:10.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:10.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:10.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.194868 25485 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:10.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.203423 25485 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.203996 25485 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.363236 25485 rpc_server.cc:307] RPC server started. Bound to: 127.23.215.253:36309
20:01:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.363382 25622 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.215.253:36309 every 8 connection(s)
20:01:10.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:10.370 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.23.215.252:40057
20:01:10.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:01:10.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.367120 25485 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-1/data/info.pb
20:01:10.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.368705 24415 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu as pid 25485
20:01:10.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.378762 24415 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:01:10.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu
20:01:10.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/wal
20:01:10.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/data
20:01:10.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:10.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:10.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:10.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:10.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:10.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:10.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/logs
20:01:10.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/data/info.pb
20:01:10.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:10.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.215.252:40057
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.215.252
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34117
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.215.252
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.215.212:40571
20:01:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057
20:01:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34355/jwks.json
20:01:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:01:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:10.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.397331 25623 sys_catalog.cc:263] Verifying existing consensus state
20:01:10.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.406453 25623 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990: Bootstrap starting.
20:01:10.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:10 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564870, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:10.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.418139 25034 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.473706 25623 log.cc:826] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990: Log is configured to *not* fsync() on all Append() calls
20:01:10.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.538748 25623 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:01:10.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.540745 25623 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990: Bootstrap complete.
20:01:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:10 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564870, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.194@KRBTEST.COM for kudu/127.23.215.253@KRBTEST.COM
20:01:10.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.573472 25173 heartbeater.cc:344] Connected to a master server at 127.23.215.253:36309
20:01:10.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.570576 25623 raft_consensus.cc:359] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:01:10.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.576817 25623 raft_consensus.cc:740] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 14182101312f479992e2922ed266d990, State: Initialized, Role: FOLLOWER
20:01:10.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.578446 25623 consensus_queue.cc:260] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:01:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.599247 25631 sys_catalog.cc:455] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:01:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.600059 25631 sys_catalog.cc:458] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [sys.catalog]: This master's current role is: FOLLOWER
20:01:10.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.611799 25623 sys_catalog.cc:565] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [sys.catalog]: configured and running, proceeding with master startup.
20:01:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.697415 25648 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:01:10.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.708971 25647 raft_consensus.cc:493] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:01:10.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.710139 25647 raft_consensus.cc:515] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:01:10.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.737497 25647 leader_election.cc:290] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 14182101312f479992e2922ed266d990 (127.23.215.253:36309), 426096099be0496fbc843c8b48c09fa4 (127.23.215.252:40057)
20:01:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:10 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564870, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.253@KRBTEST.COM for kudu/127.23.215.254@KRBTEST.COM
20:01:10.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:10 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564867, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.254@KRBTEST.COM for kudu/127.23.215.253@KRBTEST.COM
20:01:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:10.782146 25417 leader_election.cc:336] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 426096099be0496fbc843c8b48c09fa4 (127.23.215.252:40057): Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:01:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.791662 25588 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2882c6717446405899962890e037a3db" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "14182101312f479992e2922ed266d990" is_pre_election: true
20:01:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.792649 25588 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2882c6717446405899962890e037a3db in term 1.
20:01:10.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.795190 25418 leader_election.cc:304] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 14182101312f479992e2922ed266d990, 2882c6717446405899962890e037a3db; no voters: 426096099be0496fbc843c8b48c09fa4
20:01:10.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.796321 25647 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 1 FOLLOWER]: Leader pre-election won for term 2
20:01:10.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.796679 25647 raft_consensus.cc:493] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:01:10.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.796998 25647 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 1 FOLLOWER]: Advancing to term 2
20:01:10.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.806936 25647 raft_consensus.cc:515] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:01:10.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.813503 25588 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2882c6717446405899962890e037a3db" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "14182101312f479992e2922ed266d990"
20:01:10.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.814112 25588 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [term 1 FOLLOWER]: Advancing to term 2
20:01:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.824906 25588 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2882c6717446405899962890e037a3db in term 2.
20:01:10.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:10 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564870, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.253@KRBTEST.COM for kudu/127.23.215.253@KRBTEST.COM
20:01:10.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:10.836927 25417 leader_election.cc:336] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 426096099be0496fbc843c8b48c09fa4 (127.23.215.252:40057): Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111)
20:01:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.839469 25647 leader_election.cc:290] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [CANDIDATE]: Term 2 election: Requested vote from peers 14182101312f479992e2922ed266d990 (127.23.215.253:36309), 426096099be0496fbc843c8b48c09fa4 (127.23.215.252:40057)
20:01:10.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.842054 25418 leader_election.cc:304] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 14182101312f479992e2922ed266d990, 2882c6717446405899962890e037a3db; no voters: 426096099be0496fbc843c8b48c09fa4
20:01:10.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.843000 25647 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 2 FOLLOWER]: Leader election won for term 2
20:01:10.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.853904 25647 raft_consensus.cc:697] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [term 2 LEADER]: Becoming Leader. State: Replica: 2882c6717446405899962890e037a3db, State: Running, Role: LEADER
20:01:10.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.855312 25647 consensus_queue.cc:237] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:01:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.900247 25655 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2882c6717446405899962890e037a3db. Latest consensus state: current_term: 2 leader_uuid: "2882c6717446405899962890e037a3db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:01:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.901484 25655 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [sys.catalog]: This master's current role is: LEADER
20:01:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.911489 25662 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
20:01:10.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.918185 25662 catalog_manager.cc:1520] Initializing Kudu cluster ID...
20:01:10.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.859965 25485 master_runner.cc:187] Error getting master registration for 127.23.215.252:40057: Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111),
20:01:10.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.920154 25485 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:01:10.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.926684 25662 catalog_manager.cc:1295] Loaded cluster ID: c813f4ceca5b42d7a69c90bdd53b0297
20:01:10.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.928190 25662 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
20:01:10.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.933118 25662 catalog_manager.cc:1540] Loading token signing keys...
20:01:10.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.937611 25357 heartbeater.cc:344] Connected to a master server at 127.23.215.253:36309
20:01:10.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.939563 25662 catalog_manager.cc:6103] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db: Loaded TSK: 0
20:01:10.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.941134 25662 catalog_manager.cc:1550] Initializing in-progress tserver states...
20:01:10.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.944108 25574 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ba524228f99842d688a1b28b407d61bc" instance_seqno: 1786564864518653) as {username='kudu', principal='kudu/127.23.215.195@KRBTEST.COM'} at 127.23.215.195:53491; Asking this server to re-register.
20:01:10.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.945756 25357 heartbeater.cc:461] Registering TS with master...
20:01:10.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:10.949340 25574 ts_manager.cc:194] Registered new tserver with Master: ba524228f99842d688a1b28b407d61bc (127.23.215.195:36545)
20:01:11.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:11 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564865, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.193@KRBTEST.COM for kudu/127.23.215.253@KRBTEST.COM
20:01:11.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.029242 24997 heartbeater.cc:344] Connected to a master server at 127.23.215.253:36309
20:01:11.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.032047 25574 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "6da5284a5f934a50ab8431e9ee94145f" instance_seqno: 1786564857268799) as {username='kudu', principal='kudu/127.23.215.193@KRBTEST.COM'} at 127.23.215.193:44207; Asking this server to re-register.
20:01:11.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.033947 24997 heartbeater.cc:461] Registering TS with master...
20:01:11.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.037039 25574 ts_manager.cc:194] Registered new tserver with Master: 6da5284a5f934a50ab8431e9ee94145f (127.23.215.193:34191)
20:01:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:11.198812 25625 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:11.199697 25625 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:11.201007 25625 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:11.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:11.269740 25417 consensus_peers.cc:597] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db -> Peer 426096099be0496fbc843c8b48c09fa4 (127.23.215.252:40057): Couldn't send request to peer 426096099be0496fbc843c8b48c09fa4. Status: Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:01:11.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:11.272110 25625 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:11.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:11.272464 25625 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:01:11.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:11.273022 25625 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:11.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:11.273448 25625 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:11.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:11.273777 25625 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:01:11.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:11.275635 25625 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:11.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:11.275964 25625 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34355/jwks.json
20:01:11.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.343139 25588 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [term 2 FOLLOWER]: Refusing update from remote peer 2882c6717446405899962890e037a3db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
20:01:11.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.344915 25655 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [LEADER]: Connected to new peer: Peer: permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
20:01:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.354911 25625 master_runner.cc:387] Master server non-default flags:
20:01:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/logs
20:01:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.215.212:40571
20:01:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/data
20:01:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/wal
20:01:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057
20:01:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/krb5kdc/kudu.keytab
20:01:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.215.252
20:01:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.215.252:40057
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34355/jwks.json
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/data/info.pb
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.215.252
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34117
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 565fa2a0c8b8655dfe08c1dca95e4b4d9408fc9c
20:01:11.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:11.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 19:45:48 UTC on e7f111948823
20:01:11.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13311
20:01:11.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:11.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.354890 25670 sys_catalog.cc:455] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2882c6717446405899962890e037a3db. Latest consensus state: current_term: 2 leader_uuid: "2882c6717446405899962890e037a3db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:01:11.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.355545 25670 sys_catalog.cc:458] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [sys.catalog]: This master's current role is: FOLLOWER
20:01:11.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.356964 25625 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:11.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.358471 25625 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:11.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.369690 25655 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "2882c6717446405899962890e037a3db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:01:11.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.370446 25655 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [sys.catalog]: This master's current role is: LEADER
20:01:11.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.374639 25670 sys_catalog.cc:455] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "2882c6717446405899962890e037a3db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:01:11.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.375617 25670 sys_catalog.cc:458] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990 [sys.catalog]: This master's current role is: FOLLOWER
20:01:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:11.381242 25677 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:11.511744 25618 debug-util.cc:398] Leaking SignalData structure 0x7b0800071060 after lost signal to thread 25509
20:01:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.580852 25574 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "26cee1d4b3434035a2d70d2ee68c989a" instance_seqno: 1786564861055711) as {username='kudu', principal='kudu/127.23.215.194@KRBTEST.COM'} at 127.23.215.194:33235; Asking this server to re-register.
20:01:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.582834 25173 heartbeater.cc:461] Registering TS with master...
20:01:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.585997 25574 ts_manager.cc:194] Registered new tserver with Master: 26cee1d4b3434035a2d70d2ee68c989a (127.23.215.194:40345)
20:01:11.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.717559 25648 catalog_manager.cc:1295] Loaded cluster ID: c813f4ceca5b42d7a69c90bdd53b0297
20:01:11.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.717916 25648 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990: loading cluster ID for follower catalog manager: success
20:01:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.739620 25648 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990: acquiring CA information for follower catalog manager: success
20:01:11.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.746922 25648 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 14182101312f479992e2922ed266d990: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:01:11.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:11.992542 24998 heartbeater.cc:499] Master 127.23.215.254:39917 was elected leader, sending a full tablet report...
20:01:12.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:12.510835 25178 heartbeater.cc:499] Master 127.23.215.254:39917 was elected leader, sending a full tablet report...
20:01:12.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:12.762094 25376 master_runner.cc:187] Error getting master registration for 127.23.215.252:40057: Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111),
20:01:12.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:12.762823 25376 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:01:12.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:12.907788 25361 heartbeater.cc:499] Master 127.23.215.254:39917 was elected leader, sending a full tablet report...
20:01:13.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:12.781911 25676 debug-util.cc:398] Leaking SignalData structure 0x7b0800034fe0 after lost signal to thread 25625
20:01:13.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:13.659710 25625 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.275s user 0.467s sys 0.734s
20:01:13.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:13.660630 25625 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.276s user 0.467s sys 0.735s
20:01:13.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:11.386199 25678 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:13.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:13.662863 25680 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:13.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:13.667240 25679 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2278 milliseconds
20:01:13.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.667989 25625 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:13.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.669487 25625 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:13.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.672524 25625 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:13.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.673934 25625 hybrid_clock.cc:648] HybridClock initialized: now 1786564873673898 us; error 38 us; skew 500 ppm
20:01:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.674917 25625 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:13.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:13 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564873, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:13.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.681586 25625 init.cc:377] Logged in from keytab as kudu/127.23.215.252@KRBTEST.COM (short username kudu)
20:01:13.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.700821 25625 webserver.cc:533] Webserver started at http://127.23.215.252:34117/ using document root <none> and password file <none>
20:01:13.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.702088 25625 fs_manager.cc:362] Metadata directory not provided
20:01:13.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.702407 25625 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:13.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.711344 25625 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.001s
20:01:13.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.718276 25708 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:13.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.719672 25625 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.004s sys 0.001s
20:01:13.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.720050 25625 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/data,/tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/wal
20:01:13.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "426096099be0496fbc843c8b48c09fa4"
20:01:13.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 20:00:49 on dist-test-slave-1jjb"
20:01:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.722707 25625 fs_report.cc:389] FS layout report
20:01:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/wal
20:01:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/wal
20:01:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/data/data
20:01:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.780779 25625 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:13.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:13.791036 25417 consensus_peers.cc:597] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db -> Peer 426096099be0496fbc843c8b48c09fa4 (127.23.215.252:40057): Couldn't send request to peer 426096099be0496fbc843c8b48c09fa4. Status: Network error: Client connection negotiation failed: client connection to 127.23.215.252:40057: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:01:13.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.794236 25625 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.794770 25625 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:13.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.906629 25625 rpc_server.cc:307] RPC server started. Bound to: 127.23.215.252:40057
20:01:13.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.906711 25774 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.215.252:40057 every 8 connection(s)
20:01:13.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.911099 25625 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwpaMk5/test-tmp/mini-kudu-cluster13743825867606337493/master-2/data/info.pb
20:01:13.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.917941 24415 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu as pid 25625
20:01:13.918 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:13.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.928898 25775 sys_catalog.cc:263] Verifying existing consensus state
20:01:13.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.937772 25775 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4: Bootstrap starting.
20:01:13.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:13 dist-test-slave-1jjb krb5kdc[24428](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564873, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:13.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:13.981370 25214 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:14.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.018867 25775 log.cc:826] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4: Log is configured to *not* fsync() on all Append() calls
20:01:14.029 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.215.254:39917(127.23.215.254:39917)] connecting to peer
20:01:14.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:14 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564865, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.193@KRBTEST.COM for kudu/127.23.215.252@KRBTEST.COM
20:01:14.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.104051 25775 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:01:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.105104 25775 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4: Bootstrap complete.
20:01:14.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:14 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564873, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.195@KRBTEST.COM for kudu/127.23.215.252@KRBTEST.COM
20:01:14.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.143441 24996 heartbeater.cc:344] Connected to a master server at 127.23.215.252:40057
20:01:14.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.146005 25356 heartbeater.cc:344] Connected to a master server at 127.23.215.252:40057
20:01:14.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.173133 25775 raft_consensus.cc:359] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:01:14.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.176959 25775 raft_consensus.cc:740] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 426096099be0496fbc843c8b48c09fa4, State: Initialized, Role: FOLLOWER
20:01:14.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.178114 25775 consensus_queue.cc:260] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } }
20:01:14.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.191052 25775 sys_catalog.cc:565] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [sys.catalog]: configured and running, proceeding with master startup.
20:01:14.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.193111 25786 sys_catalog.cc:455] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:01:14.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.194159 25786 sys_catalog.cc:458] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [sys.catalog]: This master's current role is: FOLLOWER
20:01:14.211 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.215.253:36309(127.23.215.253:36309)] connecting to peer
20:01:14.228 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.215.252:40057(127.23.215.252:40057)] connecting to peer
20:01:14.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:14 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564867, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.254@KRBTEST.COM for kudu/127.23.215.252@KRBTEST.COM
20:01:14.269 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.23.215.253:36309(127.23.215.253:36309)] Successfully connected to peer
20:01:14.259 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.23.215.254:39917(127.23.215.254:39917)] Successfully connected to peer
20:01:14.289 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.23.215.252:40057(127.23.215.252:40057)] Successfully connected to peer
20:01:14.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.321419 25803 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:01:14.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.326568 25745 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [term 1 FOLLOWER]: Advancing to term 2
20:01:14.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:14 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564873, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.252@KRBTEST.COM for kudu/127.23.215.254@KRBTEST.COM
20:01:14.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.504491 25786 sys_catalog.cc:455] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2882c6717446405899962890e037a3db. Latest consensus state: current_term: 2 leader_uuid: "2882c6717446405899962890e037a3db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:01:14.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.505230 25786 sys_catalog.cc:458] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [sys.catalog]: This master's current role is: FOLLOWER
20:01:14.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.505743 25786 sys_catalog.cc:455] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "2882c6717446405899962890e037a3db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:01:14.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.506327 25786 sys_catalog.cc:458] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4 [sys.catalog]: This master's current role is: FOLLOWER
20:01:14.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:14 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564873, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.252@KRBTEST.COM for kudu/127.23.215.253@KRBTEST.COM
20:01:14.522 [WARN - kudu-nio-2] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
20:01:14.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:14 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564873, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.252@KRBTEST.COM for kudu/127.23.215.252@KRBTEST.COM
20:01:14.645 [DEBUG - kudu-nio-1] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.23.215.253
20:01:14.648 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.23.215.252
20:01:14.651 [DEBUG - kudu-nio-0] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.23.215.254
20:01:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 20:01:14 dist-test-slave-1jjb krb5kdc[24428](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786564870, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.215.194@KRBTEST.COM for kudu/127.23.215.252@KRBTEST.COM
20:01:14.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.707826 25172 heartbeater.cc:344] Connected to a master server at 127.23.215.252:40057
20:01:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.722729 25733 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "26cee1d4b3434035a2d70d2ee68c989a" instance_seqno: 1786564861055711) as {username='kudu', principal='kudu/127.23.215.194@KRBTEST.COM'} at 127.23.215.194:56737; Asking this server to re-register.
20:01:14.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.727923 25172 heartbeater.cc:461] Registering TS with master...
20:01:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.734429 25733 ts_manager.cc:194] Registered new tserver with Master: 26cee1d4b3434035a2d70d2ee68c989a (127.23.215.194:40345)
20:01:14.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.775745 25647 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "2882c6717446405899962890e037a3db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2882c6717446405899962890e037a3db" member_type: VOTER last_known_addr { host: "127.23.215.254" port: 39917 } } peers { permanent_uuid: "14182101312f479992e2922ed266d990" member_type: VOTER last_known_addr { host: "127.23.215.253" port: 36309 } } peers { permanent_uuid: "426096099be0496fbc843c8b48c09fa4" member_type: VOTER last_known_addr { host: "127.23.215.252" port: 40057 } } }
20:01:14.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:14.779425 25647 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2882c6717446405899962890e037a3db [sys.catalog]: This master's current role is: LEADER
20:01:15.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:15.154122 25733 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ba524228f99842d688a1b28b407d61bc" instance_seqno: 1786564864518653) as {username='kudu', principal='kudu/127.23.215.195@KRBTEST.COM'} at 127.23.215.195:50223; Asking this server to re-register.
20:01:15.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:15.156191 25732 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "6da5284a5f934a50ab8431e9ee94145f" instance_seqno: 1786564857268799) as {username='kudu', principal='kudu/127.23.215.193@KRBTEST.COM'} at 127.23.215.193:55261; Asking this server to re-register.
20:01:15.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:15.157114 25356 heartbeater.cc:461] Registering TS with master...
20:01:15.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:15.159304 24996 heartbeater.cc:461] Registering TS with master...
20:01:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:15.165766 25732 ts_manager.cc:194] Registered new tserver with Master: 6da5284a5f934a50ab8431e9ee94145f (127.23.215.193:34191)
20:01:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:15.166769 25733 ts_manager.cc:194] Registered new tserver with Master: ba524228f99842d688a1b28b407d61bc (127.23.215.195:36545)
20:01:15.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.215.253:36309(127.23.215.253:36309)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
20:01:15.226 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.215.253:36309: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
20:01:15.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:15.234527 25797 negotiation.cc:336] Failed RPC negotiation. Trace:
20:01:15.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.261540 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:56362 (local address 127.23.215.253:36309)
20:01:15.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.318628 (+ 57088us) server_negotiation.cc:207] Beginning negotiation
20:01:15.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.318669 (+ 41us) server_negotiation.cc:400] Waiting for connection header
20:01:15.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.448821 (+130152us) server_negotiation.cc:408] Connection header received
20:01:15.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.498349 (+ 49528us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:01:15.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.498379 (+ 30us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:01:15.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.499945 (+ 1566us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:01:15.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.500167 (+ 222us) server_negotiation.cc:227] Negotiated authn=SASL
20:01:15.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.965979 (+465812us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
20:01:15.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.970008 (+ 4029us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
20:01:15.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:15.212874 (+242866us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
20:01:15.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:15.213488 (+ 614us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
20:01:15.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:15.229976 (+ 16488us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56362: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:56362): Cannot send after transport endpoint shutdown (error 108)
20:01:15.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":56131,"spinlock_wait_cycles":117120,"thread_start_us":43486,"threads_started":1}
20:01:15.241 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.23.215.253:36309(127.23.215.253:36309)] handling channelInactive
20:01:15.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.215.254:39917(127.23.215.254:39917)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
20:01:15.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.215.254:39917: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
20:01:15.247 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.23.215.254:39917(127.23.215.254:39917)] handling channelInactive
20:01:15.248 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.23.215.252:40057(127.23.215.252:40057)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
20:01:15.248 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.23.215.252:40057: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
20:01:15.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:15.249204 25795 negotiation.cc:336] Failed RPC negotiation. Trace:
20:01:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.254183 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:33924 (local address 127.23.215.254:39917)
20:01:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.323893 (+ 69710us) server_negotiation.cc:207] Beginning negotiation
20:01:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.323921 (+ 28us) server_negotiation.cc:400] Waiting for connection header
20:01:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.447426 (+123505us) server_negotiation.cc:408] Connection header received
20:01:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.497643 (+ 50217us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:01:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.497676 (+ 33us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:01:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.497931 (+ 255us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:01:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.498125 (+ 194us) server_negotiation.cc:227] Negotiated authn=SASL
20:01:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.964902 (+466777us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
20:01:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.967267 (+ 2365us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
20:01:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:15.216269 (+249002us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
20:01:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:15.216850 (+ 581us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
20:01:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:15.247480 (+ 30630us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:33924: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:33924): Cannot send after transport endpoint shutdown (error 108)
20:01:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":65175,"thread_start_us":68092,"threads_started":1}
20:01:15.253 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1330), Traces: [0ms] refreshing cache from master, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.215.254:39917, [275ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.215.253:36309, [292ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.215.252:40057, [1290ms] Sub RPC ConnectToMaster: received response from server master-127.23.215.253:36309: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1309ms] Sub RPC ConnectToMaster: received response from server master-127.23.215.254:39917: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1312ms] Sub RPC ConnectToMaster: received response from server master-127.23.215.252:40057: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
20:01:15.269 [DEBUG - kudu-nio-2] (KuduRpc.java:274) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1346), Traces: [0ms] refreshing cache from master, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.215.254:39917, [275ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.215.253:36309, [292ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.215.252:40057, [1290ms] Sub RPC ConnectToMaster: received response from server master-127.23.215.253:36309: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1309ms] Sub RPC ConnectToMaster: received response from server master-127.23.215.254:39917: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1312ms] Sub RPC ConnectToMaster: received response from server master-127.23.215.252:40057: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) with no deferred attached!
20:01:15.273 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.23.215.254:39917(127.23.215.254:39917)] disconnecting while in state TERMINATED
20:01:15.276 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.23.215.253:36309(127.23.215.253:36309)] disconnecting while in state TERMINATED
20:01:15.276 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.23.215.252:40057(127.23.215.252:40057)] disconnecting while in state TERMINATED
20:01:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:15.284040 25784 negotiation.cc:336] Failed RPC negotiation. Trace:
20:01:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.281086 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:44842 (local address 127.23.215.252:40057)
20:01:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.281976 (+ 890us) server_negotiation.cc:207] Beginning negotiation
20:01:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.282003 (+ 27us) server_negotiation.cc:400] Waiting for connection header
20:01:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.446492 (+164489us) server_negotiation.cc:408] Connection header received
20:01:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.497378 (+ 50886us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:01:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.497408 (+ 30us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:01:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.497635 (+ 227us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:01:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.497786 (+ 151us) server_negotiation.cc:227] Negotiated authn=SASL
20:01:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.966730 (+468944us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
20:01:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:14.968388 (+ 1658us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
20:01:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:15.210424 (+242036us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
20:01:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:15.211076 (+ 652us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
20:01:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 20:01:15.281335 (+ 70259us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44842: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:44842): Cannot send after transport endpoint shutdown (error 108)
20:01:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":180}
20:01:15.285 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:01:15.295 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:01:15.295 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.23.215.252:40057(127.23.215.252:40057)] handling channelInactive
20:01:15.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:15.314908 24415 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu with pid 24802
20:01:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:15.337605 23820 debug-util.cc:398] Leaking SignalData structure 0x7b08000aba40 after lost signal to thread 23749
20:01:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 20:01:15.338495 23820 debug-util.cc:398] Leaking SignalData structure 0x7b08000708a0 after lost signal to thread 23824
20:01:15.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:15.348778 25803 catalog_manager.cc:1295] Loaded cluster ID: c813f4ceca5b42d7a69c90bdd53b0297
20:01:15.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:15.349376 25803 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4: loading cluster ID for follower catalog manager: success
20:01:15.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:15.359445 25803 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4: acquiring CA information for follower catalog manager: success
20:01:15.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:15.365775 25803 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 426096099be0496fbc843c8b48c09fa4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:01:15.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:15.449852 24415 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu with pid 25012
20:01:15.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:15.612525 24415 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu with pid 25191
20:01:15.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:15.696686 24415 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu with pid 25376
20:01:15.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:15.779392 24415 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu with pid 25485
20:01:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:15.831275 24415 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu with pid 25625
20:01:17.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T20:01:17Z chronyd exiting
20:01:18.263 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:01:18.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:18.265829 23562 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu with pid 23880
20:01:18.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:18.297891 23562 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu with pid 24072
20:01:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:18.330057 23562 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu with pid 24232
20:01:18.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:18.362589 23562 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu with pid 23577
20:01:18.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:18.393839 23562 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu with pid 23657
20:01:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 20:01:18.429803 23562 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwpaMk5/build/tsan/bin/kudu with pid 23744
20:01:19.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T20:01:19Z chronyd exiting
20:01:20.625 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
20:01:21.047 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
20:01:21.047 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.23.215.254:39917,127.23.215.253:36309,127.23.215.252:40057). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more