20:04:06.291 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:04:06.306 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task7Ze3A5/test-tmp
20:04:06.314 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:04:06.319 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task7Ze3A5/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:04:06.546 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:04:06.726 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:04:06.728 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:04:06.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:06.733287 18787 mini_oidc.cc:150] Starting JWKS server
20:04:06.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:06.737627 18787 webserver.cc:533] Webserver started at http://127.0.0.1:45381/ using document root <none> and password file <none>
20:04:06.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:06.738206 18787 mini_oidc.cc:186] Starting OIDC Discovery server
20:04:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:06.741093 18787 webserver.cc:533] Webserver started at http://127.0.0.1:36777/ using document root <none> and password file <none>
20:04:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-18T20:04:06Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:04:06.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-18T20:04:06Z Disabled control of system clock
20:04:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:06.813725 18787 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-0/wal
20:04:06.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-0/data
20:04:06.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:06.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:06.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:06.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:06.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:06.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:06.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-0/logs
20:04:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-0/data/info.pb
20:04:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:06.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:06.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:06.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:06.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:06.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:06.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:06.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:06.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:06.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:06.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:06.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.88.254:45489
20:04:06.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.88.254
20:04:06.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:06.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.88.212:43575
20:04:06.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:06.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:06.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:06.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:06.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.88.254:45489,127.18.88.253:46251,127.18.88.252:41909
20:04:06.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:06.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45381/jwks.json with env {}
20:04:07.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:07.179641 18799 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:07.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:07.180338 18799 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:07.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:07.181017 18799 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:07.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:07.227965 18799 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:07.228483 18799 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:07.228713 18799 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:07.229013 18799 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:07.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:07.229316 18799 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:07.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:07.229545 18799 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:45381/jwks.json
20:04:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:07.273834 18799 master_runner.cc:387] Master server non-default flags:
20:04:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.88.212:43575
20:04:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-0/data
20:04:07.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-0/wal
20:04:07.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.88.254:45489,127.18.88.253:46251,127.18.88.252:41909
20:04:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.88.254:45489
20:04:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45381/jwks.json
20:04:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-0/data/info.pb
20:04:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.88.254
20:04:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-0/logs
20:04:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:07.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:07.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:07.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:07.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:07.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:04:07.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:07.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:04:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:04:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:07.275813 18799 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:07.278493 18799 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:07.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:07.295905 18806 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:09.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:08.701036 18804 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 18799
20:04:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.022495 18804 kernel_stack_watchdog.cc:198] Thread 18799 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
20:04:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:04:09.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:04:09.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:04:09.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:04:09.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:07.296033 18805 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:09.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.029376 18799 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.733s user 0.007s sys 0.000s
20:04:09.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.029721 18807 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1727 milliseconds
20:04:09.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.029881 18799 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.734s user 0.007s sys 0.000s
20:04:09.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.032001 18808 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:09.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.032056 18799 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:09.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.033672 18799 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:09.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.037165 18799 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:09.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.039691 18799 hybrid_clock.cc:648] HybridClock initialized: now 1781813049039612 us; error 84 us; skew 500 ppm
20:04:09.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.040709 18799 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.048350 18799 webserver.cc:533] Webserver started at http://127.18.88.254:35873/ using document root <none> and password file <none>
20:04:09.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.049615 18799 fs_manager.cc:362] Metadata directory not provided
20:04:09.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.049841 18799 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:09.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.050261 18799 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:09.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.054992 18799 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-0/data/instance:
20:04:09.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f2aa2698ea94bafa05feca9801aa8d7"
20:04:09.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:09 on dist-test-slave-k9gl"
20:04:09.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.056150 18799 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-0/wal/instance:
20:04:09.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f2aa2698ea94bafa05feca9801aa8d7"
20:04:09.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:09 on dist-test-slave-k9gl"
20:04:09.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.065150 18799 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.011s sys 0.000s
20:04:09.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.073275 18815 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:09.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.075179 18799 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
20:04:09.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.075789 18799 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-0/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-0/wal
20:04:09.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f2aa2698ea94bafa05feca9801aa8d7"
20:04:09.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:09 on dist-test-slave-k9gl"
20:04:09.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.076301 18799 fs_report.cc:389] FS layout report
20:04:09.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:09.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-0/wal
20:04:09.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-0/wal
20:04:09.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-0/data/data
20:04:09.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:09.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:09.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:09.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:09.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:09.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:09.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:09.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:09.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:09.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:09.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:09.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.129354 18799 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:09.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.131043 18799 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:09.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.131528 18799 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:09.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.217561 18799 rpc_server.cc:307] RPC server started. Bound to: 127.18.88.254:45489
20:04:09.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.217723 18866 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.88.254:45489 every 8 connection(s)
20:04:09.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.221405 18799 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-0/data/info.pb
20:04:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.227280 18787 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 18799
20:04:09.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.227897 18787 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-0/wal/instance
20:04:09.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.230096 18867 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:04:09.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.233923 18787 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:09.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:09.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-1/wal
20:04:09.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-1/data
20:04:09.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:09.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:09.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-1/logs
20:04:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-1/data/info.pb
20:04:09.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:09.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:09.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:09.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:09.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:09.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:09.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:09.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:09.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:09.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:09.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.88.253:46251
20:04:09.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.88.253
20:04:09.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:09.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.88.212:43575
20:04:09.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:09.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.88.254:45489,127.18.88.253:46251,127.18.88.252:41909
20:04:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45381/jwks.json with env {}
20:04:09.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.249542 18867 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } has no permanent_uuid. Determining permanent_uuid...
20:04:09.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.291257 18867 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } has no permanent_uuid. Determining permanent_uuid...
20:04:09.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.295064 18817 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.88.253:46251: connect: Connection refused (error 111)
20:04:09.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.301813 18867 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.253:46251: Network error: Client connection negotiation failed: client connection to 127.18.88.253:46251: connect: Connection refused (error 111)
20:04:09.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.351583 18867 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } attempt: 1
20:04:09.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.356840 18867 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.253:46251: Network error: Client connection negotiation failed: client connection to 127.18.88.253:46251: connect: Connection refused (error 111)
20:04:09.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.425411 18867 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } attempt: 2
20:04:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.430492 18867 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.253:46251: Network error: Client connection negotiation failed: client connection to 127.18.88.253:46251: connect: Connection refused (error 111)
20:04:09.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.522123 18867 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } attempt: 3
20:04:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.527043 18867 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.253:46251: Network error: Client connection negotiation failed: client connection to 127.18.88.253:46251: connect: Connection refused (error 111)
20:04:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.586634 18869 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.587208 18869 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.587642 18869 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:09.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.623944 18869 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:09.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.624255 18869 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:09.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.624454 18869 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:09.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.624642 18869 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:09.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.624852 18869 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:09.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.624994 18869 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:45381/jwks.json
20:04:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.663456 18869 master_runner.cc:387] Master server non-default flags:
20:04:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.88.212:43575
20:04:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-1/data
20:04:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-1/wal
20:04:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:09.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.88.254:45489,127.18.88.253:46251,127.18.88.252:41909
20:04:09.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:09.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:09.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:09.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.88.253:46251
20:04:09.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:09.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:09.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:09.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45381/jwks.json
20:04:09.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:09.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:09.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-1/data/info.pb
20:04:09.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.88.253
20:04:09.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:09.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:09.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:09.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:09.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:09.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:09.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-1/logs
20:04:09.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:09.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:09.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:09.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:09.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:09.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:04:09.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:09.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:04:09.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:04:09.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:09.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.665263 18869 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:09.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.667559 18869 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:09.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.670543 18867 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } attempt: 4
20:04:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.676970 18867 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.253:46251: Network error: Client connection negotiation failed: client connection to 127.18.88.253:46251: connect: Connection refused (error 111)
20:04:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.681970 18878 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:09.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:09.976490 18867 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } attempt: 5
20:04:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.989646 18867 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.253:46251: Network error: Client connection negotiation failed: client connection to 127.18.88.253:46251: connect: Connection refused (error 111)
20:04:10.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:10.537293 18867 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } attempt: 6
20:04:10.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:10.563954 18867 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.253:46251: Network error: Client connection negotiation failed: client connection to 127.18.88.253:46251: connect: Connection refused (error 111)
20:04:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:11.086879 18877 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 18869
20:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:11.378484 18877 kernel_stack_watchdog.cc:198] Thread 18869 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
20:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:04:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:04:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:09.683082 18879 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:11.380826 18869 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.699s user 0.512s sys 1.161s
20:04:11.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:11.381677 18869 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.700s user 0.513s sys 1.161s
20:04:11.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:11.382117 18881 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:11.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:11.385161 18880 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1697 milliseconds
20:04:11.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.385299 18869 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:11.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.387109 18869 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:11.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.390296 18869 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:11.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.391813 18869 hybrid_clock.cc:648] HybridClock initialized: now 1781813051391732 us; error 79 us; skew 500 ppm
20:04:11.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.393028 18869 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:11.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.401147 18869 webserver.cc:533] Webserver started at http://127.18.88.253:43615/ using document root <none> and password file <none>
20:04:11.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.402516 18869 fs_manager.cc:362] Metadata directory not provided
20:04:11.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.402805 18869 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:11.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.403448 18869 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.408890 18869 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-1/data/instance:
20:04:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c494afa7f15413a85be38540ce1ef7f"
20:04:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:11 on dist-test-slave-k9gl"
20:04:11.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.410634 18869 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-1/wal/instance:
20:04:11.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c494afa7f15413a85be38540ce1ef7f"
20:04:11.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:11 on dist-test-slave-k9gl"
20:04:11.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.419937 18869 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.009s sys 0.001s
20:04:11.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.429082 18890 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:11.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.430802 18869 fs_manager.cc:730] Time spent opening block manager: real 0.007s user 0.002s sys 0.005s
20:04:11.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.431291 18869 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-1/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-1/wal
20:04:11.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c494afa7f15413a85be38540ce1ef7f"
20:04:11.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:11 on dist-test-slave-k9gl"
20:04:11.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.431775 18869 fs_report.cc:389] FS layout report
20:04:11.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:11.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-1/wal
20:04:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-1/wal
20:04:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-1/data/data
20:04:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:11.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.502836 18869 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:11.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.505059 18869 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:11.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.505690 18869 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:11.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.592401 18869 rpc_server.cc:307] RPC server started. Bound to: 127.18.88.253:46251
20:04:11.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.592476 18941 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.88.253:46251 every 8 connection(s)
20:04:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.596264 18869 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-1/data/info.pb
20:04:11.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.603780 18942 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:04:11.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.605516 18787 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 18869
20:04:11.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.605944 18787 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-1/wal/instance
20:04:11.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.608127 18787 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:11.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:11.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-2/wal
20:04:11.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-2/data
20:04:11.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:11.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:11.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:11.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:11.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:11.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-2/logs
20:04:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-2/data/info.pb
20:04:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:11.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:11.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:11.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:11.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.88.252:41909
20:04:11.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.88.252
20:04:11.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:11.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.88.212:43575
20:04:11.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:11.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:11.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:11.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:11.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.88.254:45489,127.18.88.253:46251,127.18.88.252:41909
20:04:11.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45381/jwks.json with env {}
20:04:11.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.619925 18942 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } has no permanent_uuid. Determining permanent_uuid...
20:04:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.624629 18867 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } attempt: 7
20:04:11.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.656350 18942 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } has no permanent_uuid. Determining permanent_uuid...
20:04:11.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.658931 18867 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } has no permanent_uuid. Determining permanent_uuid...
20:04:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:11.667512 18867 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.252:41909: Network error: Client connection negotiation failed: client connection to 127.18.88.252:41909: connect: Connection refused (error 111)
20:04:11.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.670889 18942 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } has no permanent_uuid. Determining permanent_uuid...
20:04:11.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:11.674181 18893 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.88.252:41909: connect: Connection refused (error 111)
20:04:11.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:11.679669 18942 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.252:41909: Network error: Client connection negotiation failed: client connection to 127.18.88.252:41909: connect: Connection refused (error 111)
20:04:11.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.726122 18867 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } attempt: 1
20:04:11.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.729377 18942 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } attempt: 1
20:04:11.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:11.736326 18867 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.252:41909: Network error: Client connection negotiation failed: client connection to 127.18.88.252:41909: connect: Connection refused (error 111)
20:04:11.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:11.736718 18942 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.252:41909: Network error: Client connection negotiation failed: client connection to 127.18.88.252:41909: connect: Connection refused (error 111)
20:04:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.805291 18942 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } attempt: 2
20:04:11.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:11.811237 18942 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.252:41909: Network error: Client connection negotiation failed: client connection to 127.18.88.252:41909: connect: Connection refused (error 111)
20:04:11.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.817874 18867 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } attempt: 2
20:04:11.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:11.824333 18867 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.252:41909: Network error: Client connection negotiation failed: client connection to 127.18.88.252:41909: connect: Connection refused (error 111)
20:04:11.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.902746 18942 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } attempt: 3
20:04:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:11.908046 18942 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.252:41909: Network error: Client connection negotiation failed: client connection to 127.18.88.252:41909: connect: Connection refused (error 111)
20:04:11.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:11.909884 18867 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } attempt: 3
20:04:11.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:11.915361 18867 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.252:41909: Network error: Client connection negotiation failed: client connection to 127.18.88.252:41909: connect: Connection refused (error 111)
20:04:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:11.971644 18944 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:11.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:11.972412 18944 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:11.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:11.973178 18944 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:12.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:12.007345 18944 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:12.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:12.007745 18944 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:12.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:12.008088 18944 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:12.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:12.008443 18944 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:12.008867 18944 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:12.009145 18944 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:45381/jwks.json
20:04:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:12.047536 18944 master_runner.cc:387] Master server non-default flags:
20:04:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.88.212:43575
20:04:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-2/data
20:04:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-2/wal
20:04:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.88.254:45489,127.18.88.253:46251,127.18.88.252:41909
20:04:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.88.252:41909
20:04:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45381/jwks.json
20:04:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-2/data/info.pb
20:04:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.88.252
20:04:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-2/logs
20:04:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:12.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:12.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:12.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:04:12.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:12.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:04:12.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:04:12.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:12.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:12.049337 18944 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:12.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:12.051724 18944 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:12.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:12.051548 18942 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } attempt: 4
20:04:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:12.055850 18867 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } attempt: 4
20:04:12.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:12.057611 18942 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.252:41909: Network error: Client connection negotiation failed: client connection to 127.18.88.252:41909: connect: Connection refused (error 111)
20:04:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:12.061386 18867 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.252:41909: Network error: Client connection negotiation failed: client connection to 127.18.88.252:41909: connect: Connection refused (error 111)
20:04:12.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:12.066393 18956 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:12.344980 18867 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } attempt: 5
20:04:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:12.357082 18942 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } attempt: 5
20:04:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:12.358456 18867 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.252:41909: Network error: Client connection negotiation failed: client connection to 127.18.88.252:41909: connect: Connection refused (error 111)
20:04:12.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:12.365072 18942 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.252:41909: Network error: Client connection negotiation failed: client connection to 127.18.88.252:41909: connect: Connection refused (error 111)
20:04:12.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:12.911018 18867 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } attempt: 6
20:04:12.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:12.912904 18942 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } attempt: 6
20:04:12.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:12.946542 18867 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.252:41909: Network error: Client connection negotiation failed: client connection to 127.18.88.252:41909: connect: Connection refused (error 111)
20:04:12.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:12.948683 18942 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.88.252:41909: Network error: Client connection negotiation failed: client connection to 127.18.88.252:41909: connect: Connection refused (error 111)
20:04:13.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:12.066885 18957 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:13.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:13.392506 18959 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:13.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:13.394343 18958 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1321 milliseconds
20:04:13.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:13.397305 18944 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.331s user 0.536s sys 0.783s
20:04:13.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:13.397660 18944 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.331s user 0.536s sys 0.783s
20:04:13.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.397872 18944 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:13.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.399636 18944 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:13.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.403218 18944 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:13.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.404714 18944 hybrid_clock.cc:648] HybridClock initialized: now 1781813053404636 us; error 89 us; skew 500 ppm
20:04:13.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.405915 18944 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.415917 18944 webserver.cc:533] Webserver started at http://127.18.88.252:36341/ using document root <none> and password file <none>
20:04:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.417367 18944 fs_manager.cc:362] Metadata directory not provided
20:04:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.417678 18944 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:13.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.418300 18944 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.423604 18944 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-2/data/instance:
20:04:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "814c2af2dee14bd4a5f7760ce632f09e"
20:04:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:13 on dist-test-slave-k9gl"
20:04:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.425271 18944 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-2/wal/instance:
20:04:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "814c2af2dee14bd4a5f7760ce632f09e"
20:04:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:13 on dist-test-slave-k9gl"
20:04:13.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.435586 18944 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.000s sys 0.008s
20:04:13.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.445564 18971 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.447682 18944 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.001s sys 0.003s
20:04:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.448314 18944 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-2/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-2/wal
20:04:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "814c2af2dee14bd4a5f7760ce632f09e"
20:04:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:13 on dist-test-slave-k9gl"
20:04:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.448969 18944 fs_report.cc:389] FS layout report
20:04:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-2/wal
20:04:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-2/wal
20:04:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-2/data/data
20:04:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:13.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:13.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:13.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:13.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.540429 18944 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.542461 18944 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.543103 18944 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:13.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.637594 18944 rpc_server.cc:307] RPC server started. Bound to: 127.18.88.252:41909
20:04:13.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.637709 19022 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.88.252:41909 every 8 connection(s)
20:04:13.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.641355 18944 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-2/data/info.pb
20:04:13.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.645161 18787 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 18944
20:04:13.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.645787 18787 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/master-2/wal/instance
20:04:13.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.650147 19023 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:04:13.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.670315 19023 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } has no permanent_uuid. Determining permanent_uuid...
20:04:13.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.698838 19023 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } has no permanent_uuid. Determining permanent_uuid...
20:04:13.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.710727 19023 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } has no permanent_uuid. Determining permanent_uuid...
20:04:13.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.729888 19023 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e: Bootstrap starting.
20:04:13.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.737303 19023 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e: Neither blocks nor log segments found. Creating new log.
20:04:13.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.739626 19023 log.cc:826] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e: Log is configured to *not* fsync() on all Append() calls
20:04:13.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.745227 19023 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e: No bootstrap required, opened a new log
20:04:13.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.766705 19023 raft_consensus.cc:359] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } }
20:04:13.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.767449 19023 raft_consensus.cc:385] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:04:13.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.767738 19023 raft_consensus.cc:740] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 814c2af2dee14bd4a5f7760ce632f09e, State: Initialized, Role: FOLLOWER
20:04:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.768654 19023 consensus_queue.cc:260] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } }
20:04:13.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.773202 19031 sys_catalog.cc:455] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } } }
20:04:13.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.773985 19031 sys_catalog.cc:458] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [sys.catalog]: This master's current role is: FOLLOWER
20:04:13.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.774487 19023 sys_catalog.cc:565] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [sys.catalog]: configured and running, proceeding with master startup.
20:04:13.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:13.800439 18977 tablet.cc:2406] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:04:13.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:13.804173 19042 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:04:13.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:13.804647 19042 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:04:13.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.798494 18944 master_runner.cc:187] Error getting master registration for 127.18.88.254:45489: OK, instance_id { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" instance_seqno: 1781813049192671 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:04:13.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.828984 18944 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:04:13.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:13.980212 18867 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } attempt: 7
20:04:14.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.002218 18867 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7: Bootstrap starting.
20:04:14.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.008816 18867 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7: Neither blocks nor log segments found. Creating new log.
20:04:14.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.009373 18942 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } attempt: 7
20:04:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.011536 18867 log.cc:826] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7: Log is configured to *not* fsync() on all Append() calls
20:04:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.018070 18867 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7: No bootstrap required, opened a new log
20:04:14.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.033578 18942 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f: Bootstrap starting.
20:04:14.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.040383 18942 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f: Neither blocks nor log segments found. Creating new log.
20:04:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.043247 18942 log.cc:826] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f: Log is configured to *not* fsync() on all Append() calls
20:04:14.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.054441 18942 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f: No bootstrap required, opened a new log
20:04:14.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.056972 18867 raft_consensus.cc:359] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } }
20:04:14.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.057852 18867 raft_consensus.cc:385] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:04:14.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.058156 18867 raft_consensus.cc:740] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f2aa2698ea94bafa05feca9801aa8d7, State: Initialized, Role: FOLLOWER
20:04:14.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.059506 18867 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } }
20:04:14.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.062852 19048 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } } }
20:04:14.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.064282 19048 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7 [sys.catalog]: This master's current role is: FOLLOWER
20:04:14.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.065213 18867 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7 [sys.catalog]: configured and running, proceeding with master startup.
20:04:14.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.087814 18942 raft_consensus.cc:359] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } }
20:04:14.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.089843 18942 raft_consensus.cc:385] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:04:14.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.091348 18942 raft_consensus.cc:740] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1c494afa7f15413a85be38540ce1ef7f, State: Initialized, Role: FOLLOWER
20:04:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.094034 18942 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } }
20:04:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.109754 19060 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } } }
20:04:14.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.111127 19060 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f [sys.catalog]: This master's current role is: FOLLOWER
20:04:14.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.114751 18942 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f [sys.catalog]: configured and running, proceeding with master startup.
20:04:14.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.142758 19031 raft_consensus.cc:493] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:04:14.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.143414 19031 raft_consensus.cc:515] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } }
20:04:14.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.101744 18799 master_runner.cc:187] Error getting master registration for 127.18.88.253:46251: OK, instance_id { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" instance_seqno: 1781813051569144 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:04:14.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.145135 18799 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:04:14.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.151770 18917 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "814c2af2dee14bd4a5f7760ce632f09e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1c494afa7f15413a85be38540ce1ef7f" is_pre_election: true
20:04:14.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.152915 18917 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 814c2af2dee14bd4a5f7760ce632f09e in term 0.
20:04:14.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.155798 18973 leader_election.cc:304] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1c494afa7f15413a85be38540ce1ef7f, 814c2af2dee14bd4a5f7760ce632f09e; no voters:
20:04:14.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:14.146409 18821 tablet.cc:2406] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:04:14.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.148672 18842 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "814c2af2dee14bd4a5f7760ce632f09e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" is_pre_election: true
20:04:14.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.160984 18842 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 814c2af2dee14bd4a5f7760ce632f09e in term 0.
20:04:14.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.164441 19031 leader_election.cc:290] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6f2aa2698ea94bafa05feca9801aa8d7 (127.18.88.254:45489), 1c494afa7f15413a85be38540ce1ef7f (127.18.88.253:46251)
20:04:14.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.165143 19031 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [term 0 FOLLOWER]: Leader pre-election won for term 1
20:04:14.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.165527 19031 raft_consensus.cc:493] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:04:14.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.165850 19031 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [term 0 FOLLOWER]: Advancing to term 1
20:04:14.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:14.151839 19059 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:04:14.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:14.168308 19059 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:04:14.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.186918 19031 raft_consensus.cc:515] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } }
20:04:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.198202 18842 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "814c2af2dee14bd4a5f7760ce632f09e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f2aa2698ea94bafa05feca9801aa8d7"
20:04:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.198904 18842 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7 [term 0 FOLLOWER]: Advancing to term 1
20:04:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.209702 18842 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 814c2af2dee14bd4a5f7760ce632f09e in term 1.
20:04:14.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.212652 18917 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "814c2af2dee14bd4a5f7760ce632f09e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1c494afa7f15413a85be38540ce1ef7f"
20:04:14.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.213399 18917 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f [term 0 FOLLOWER]: Advancing to term 1
20:04:14.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.222589 18917 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 814c2af2dee14bd4a5f7760ce632f09e in term 1.
20:04:14.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.223646 19031 leader_election.cc:290] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [CANDIDATE]: Term 1 election: Requested vote from peers 6f2aa2698ea94bafa05feca9801aa8d7 (127.18.88.254:45489), 1c494afa7f15413a85be38540ce1ef7f (127.18.88.253:46251)
20:04:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.226089 18975 leader_election.cc:304] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6f2aa2698ea94bafa05feca9801aa8d7, 814c2af2dee14bd4a5f7760ce632f09e; no voters:
20:04:14.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.230522 19068 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [term 1 FOLLOWER]: Leader election won for term 1
20:04:14.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.239308 19068 raft_consensus.cc:697] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [term 1 LEADER]: Becoming Leader. State: Replica: 814c2af2dee14bd4a5f7760ce632f09e, State: Running, Role: LEADER
20:04:14.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.241302 19068 consensus_queue.cc:237] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } }
20:04:14.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.257761 19074 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:04:14.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.259970 19031 sys_catalog.cc:455] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 814c2af2dee14bd4a5f7760ce632f09e. Latest consensus state: current_term: 1 leader_uuid: "814c2af2dee14bd4a5f7760ce632f09e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } } }
20:04:14.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.260918 19031 sys_catalog.cc:458] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [sys.catalog]: This master's current role is: LEADER
20:04:14.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:14.267735 18896 tablet.cc:2406] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.272745 19076 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
20:04:14.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.281307 19076 catalog_manager.cc:1519] Initializing Kudu cluster ID...
20:04:14.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.308533 18917 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f [term 1 FOLLOWER]: Refusing update from remote peer 814c2af2dee14bd4a5f7760ce632f09e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:04:14.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.311719 19031 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:04:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.318639 18842 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7 [term 1 FOLLOWER]: Refusing update from remote peer 814c2af2dee14bd4a5f7760ce632f09e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:04:14.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.321952 19068 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:04:14.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.351635 19060 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 814c2af2dee14bd4a5f7760ce632f09e. Latest consensus state: current_term: 1 leader_uuid: "814c2af2dee14bd4a5f7760ce632f09e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } } }
20:04:14.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.352494 19060 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f [sys.catalog]: This master's current role is: FOLLOWER
20:04:14.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.355412 19077 mvcc.cc:204] Tried to move back new op lower bound from 7298306270422695936 to 7298306270190542848. Current Snapshot: MvccSnapshot[applied={T|T < 7298306270422695936}]
20:04:14.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.360780 19073 sys_catalog.cc:455] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "814c2af2dee14bd4a5f7760ce632f09e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } } }
20:04:14.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.361769 19073 sys_catalog.cc:458] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [sys.catalog]: This master's current role is: LEADER
20:04:14.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.365703 19076 catalog_manager.cc:1382] Generated new cluster ID: f28379ec9fe64f6a9fab81092f5e2309
20:04:14.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.366058 19076 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
20:04:14.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.369788 19060 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "814c2af2dee14bd4a5f7760ce632f09e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } } }
20:04:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.370512 19060 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f [sys.catalog]: This master's current role is: FOLLOWER
20:04:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.371150 19048 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 814c2af2dee14bd4a5f7760ce632f09e. Latest consensus state: current_term: 1 leader_uuid: "814c2af2dee14bd4a5f7760ce632f09e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } } }
20:04:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.373250 19048 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7 [sys.catalog]: This master's current role is: FOLLOWER
20:04:14.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.377938 19073 sys_catalog.cc:455] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "814c2af2dee14bd4a5f7760ce632f09e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } } }
20:04:14.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.378715 19073 sys_catalog.cc:458] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e [sys.catalog]: This master's current role is: LEADER
20:04:14.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.395244 19048 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "814c2af2dee14bd4a5f7760ce632f09e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f2aa2698ea94bafa05feca9801aa8d7" member_type: VOTER last_known_addr { host: "127.18.88.254" port: 45489 } } peers { permanent_uuid: "1c494afa7f15413a85be38540ce1ef7f" member_type: VOTER last_known_addr { host: "127.18.88.253" port: 46251 } } peers { permanent_uuid: "814c2af2dee14bd4a5f7760ce632f09e" member_type: VOTER last_known_addr { host: "127.18.88.252" port: 41909 } } }
20:04:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.396039 19048 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7 [sys.catalog]: This master's current role is: FOLLOWER
20:04:14.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.420905 19076 catalog_manager.cc:1405] Generated new certificate authority record
20:04:14.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.423074 19076 catalog_manager.cc:1539] Loading token signing keys...
20:04:14.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.450555 19076 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 814c2af2dee14bd4a5f7760ce632f09e: Generated new TSK 0
20:04:14.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.451776 19076 catalog_manager.cc:1549] Initializing in-progress tserver states...
20:04:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:14.470427 18787 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-0/wal
20:04:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-0/data
20:04:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-0/logs
20:04:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-0/data/info.pb
20:04:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.88.193:0
20:04:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.88.193
20:04:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.88.193
20:04:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:14.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.88.254:45489,127.18.88.253:46251,127.18.88.252:41909
20:04:14.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.88.212:43575
20:04:14.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:14.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:14.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:04:15.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:15.000774 19086 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:15.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:15.002311 19086 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:15.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:15.003046 19086 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:15.065852 19086 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:15.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:15.067231 19086 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.18.88.193
20:04:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:15.133868 19086 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.88.212:43575
20:04:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-0/data
20:04:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-0/wal
20:04:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.88.193:0
20:04:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-0/data/info.pb
20:04:15.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.88.193
20:04:15.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:15.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.88.254:45489,127.18.88.253:46251,127.18.88.252:41909
20:04:15.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:15.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:15.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:15.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:15.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:15.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.88.193
20:04:15.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-0/logs
20:04:15.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:15.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:15.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:15.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:15.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:15.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:04:15.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:15.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:04:15.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:04:15.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:15.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:15.135653 19086 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:15.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:15.137890 19086 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:15.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:15.156725 19093 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:15.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:15.174036 19059 catalog_manager.cc:1294] Loaded cluster ID: f28379ec9fe64f6a9fab81092f5e2309
20:04:15.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:15.174809 19059 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7: loading cluster ID for follower catalog manager: success
20:04:15.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:15.182188 19059 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7: acquiring CA information for follower catalog manager: success
20:04:15.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:15.189131 19059 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 6f2aa2698ea94bafa05feca9801aa8d7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:04:15.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:15.281642 19074 catalog_manager.cc:1294] Loaded cluster ID: f28379ec9fe64f6a9fab81092f5e2309
20:04:15.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:15.282017 19074 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f: loading cluster ID for follower catalog manager: success
20:04:15.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:15.306748 19074 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f: acquiring CA information for follower catalog manager: success
20:04:15.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:15.323052 19074 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 1c494afa7f15413a85be38540ce1ef7f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:04:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:15.157436 19092 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:16.559454 19091 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 19086
20:04:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:17.935143 19086 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.781s user 1.035s sys 1.586s
20:04:17.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:17.937217 19086 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.784s user 1.036s sys 1.587s
20:04:17.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:17.945624 19086 server_base.cc:1061] running on GCE node
20:04:17.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:17.947772 19101 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:17.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:17.949505 19086 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:17.952724 19086 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:17.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:17.954948 19086 hybrid_clock.cc:648] HybridClock initialized: now 1781813057954818 us; error 182 us; skew 500 ppm
20:04:17.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:17.956326 19086 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:17.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:17.966426 19086 webserver.cc:533] Webserver started at http://127.18.88.193:45925/ using document root <none> and password file <none>
20:04:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:17.968025 19086 fs_manager.cc:362] Metadata directory not provided
20:04:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:17.968369 19086 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:17.969223 19086 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:17.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:17.978211 19086 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-0/data/instance:
20:04:17.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8abe7e78f90146e3ac1b273e855ed3cd"
20:04:17.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:17 on dist-test-slave-k9gl"
20:04:17.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:17.980342 19086 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-0/wal/instance:
20:04:17.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8abe7e78f90146e3ac1b273e855ed3cd"
20:04:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:17 on dist-test-slave-k9gl"
20:04:17.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:17.993994 19086 fs_manager.cc:696] Time spent creating directory manager: real 0.013s user 0.005s sys 0.005s
20:04:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.003424 19106 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.005306 19086 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.005s sys 0.001s
20:04:18.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.005990 19086 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-0/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-0/wal
20:04:18.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8abe7e78f90146e3ac1b273e855ed3cd"
20:04:18.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:17 on dist-test-slave-k9gl"
20:04:18.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.006498 19086 fs_report.cc:389] FS layout report
20:04:18.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:18.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-0/wal
20:04:18.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-0/wal
20:04:18.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-0/data/data
20:04:18.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:18.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:18.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:18.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:18.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:18.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:18.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:18.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:18.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:18.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:18.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.088708 19086 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:18.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.091198 19086 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:18.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.091938 19086 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.096400 19086 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:18.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.103334 19086 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:04:18.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.103706 19086 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:04:18.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.104043 19086 ts_tablet_manager.cc:616] Registered 0 tablets
20:04:18.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.104271 19086 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:04:18.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.371166 19086 rpc_server.cc:307] RPC server started. Bound to: 127.18.88.193:36727
20:04:18.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.371346 19218 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.88.193:36727 every 8 connection(s)
20:04:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.375365 19086 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-0/data/info.pb
20:04:18.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.384958 18787 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 19086
20:04:18.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.385730 18787 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-0/wal/instance
20:04:18.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.426283 18787 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-1/wal
20:04:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-1/data
20:04:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-1/logs
20:04:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-1/data/info.pb
20:04:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.88.194:0
20:04:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.88.194
20:04:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.88.194
20:04:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.88.254:45489,127.18.88.253:46251,127.18.88.252:41909
20:04:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.88.212:43575
20:04:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:04:18.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.458320 19219 heartbeater.cc:344] Connected to a master server at 127.18.88.252:41909
20:04:18.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.459162 19219 heartbeater.cc:461] Registering TS with master...
20:04:18.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.460953 19219 heartbeater.cc:507] Master 127.18.88.252:41909 requested a full tablet report, sending...
20:04:18.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.465252 18988 ts_manager.cc:194] Registered new tserver with Master: 8abe7e78f90146e3ac1b273e855ed3cd (127.18.88.193:36727)
20:04:18.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.472363 18988 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.18.88.193:35093
20:04:18.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.505311 19225 heartbeater.cc:344] Connected to a master server at 127.18.88.254:45489
20:04:18.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.505790 19225 heartbeater.cc:461] Registering TS with master...
20:04:18.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.506583 19225 heartbeater.cc:507] Master 127.18.88.254:45489 requested a full tablet report, sending...
20:04:18.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.508924 19220 heartbeater.cc:344] Connected to a master server at 127.18.88.253:46251
20:04:18.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.509358 19220 heartbeater.cc:461] Registering TS with master...
20:04:18.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.509996 19220 heartbeater.cc:507] Master 127.18.88.253:46251 requested a full tablet report, sending...
20:04:18.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.510130 18832 ts_manager.cc:194] Registered new tserver with Master: 8abe7e78f90146e3ac1b273e855ed3cd (127.18.88.193:36727)
20:04:18.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.514417 18907 ts_manager.cc:194] Registered new tserver with Master: 8abe7e78f90146e3ac1b273e855ed3cd (127.18.88.193:36727)
20:04:18.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:18.888777 19228 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:18.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:18.889389 19228 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:18.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:18.889833 19228 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:18.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:18.931301 19228 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:18.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:18.932597 19228 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.18.88.194
20:04:18.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.980165 19228 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.88.212:43575
20:04:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-1/data
20:04:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-1/wal
20:04:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.88.194:0
20:04:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-1/data/info.pb
20:04:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.88.194
20:04:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.88.254:45489,127.18.88.253:46251,127.18.88.252:41909
20:04:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.88.194
20:04:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-1/logs
20:04:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:04:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:04:18.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:04:18.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:18.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.981930 19228 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:18.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:18.984349 19228 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:19.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:19.006837 19236 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:19.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:19.491650 19219 heartbeater.cc:499] Master 127.18.88.252:41909 was elected leader, sending a full tablet report...
20:04:20.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:19.008955 19235 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:20.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:20.408267 19238 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:20.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:20.411527 19237 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1397 milliseconds
20:04:20.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.411674 19228 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:20.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.413821 19228 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.416502 19228 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:20.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.418099 19228 hybrid_clock.cc:648] HybridClock initialized: now 1781813060418037 us; error 84 us; skew 500 ppm
20:04:20.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.419121 19228 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:20.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.427358 19228 webserver.cc:533] Webserver started at http://127.18.88.194:46667/ using document root <none> and password file <none>
20:04:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.429052 19228 fs_manager.cc:362] Metadata directory not provided
20:04:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.429425 19228 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.430006 19228 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:20.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.435035 19228 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-1/data/instance:
20:04:20.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9c779fc78f84f2db3c9f93594294bd9"
20:04:20.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:20 on dist-test-slave-k9gl"
20:04:20.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.436637 19228 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-1/wal/instance:
20:04:20.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9c779fc78f84f2db3c9f93594294bd9"
20:04:20.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:20 on dist-test-slave-k9gl"
20:04:20.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.446270 19228 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
20:04:20.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.454115 19246 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:20.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.455613 19228 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
20:04:20.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.456079 19228 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-1/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-1/wal
20:04:20.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9c779fc78f84f2db3c9f93594294bd9"
20:04:20.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:20 on dist-test-slave-k9gl"
20:04:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.456501 19228 fs_report.cc:389] FS layout report
20:04:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-1/wal
20:04:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-1/wal
20:04:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-1/data/data
20:04:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:20.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.518277 19228 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:20.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.519992 19228 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:20.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.520566 19228 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.523528 19228 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:20.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.529886 19228 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:04:20.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.530231 19228 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:04:20.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.530534 19228 ts_tablet_manager.cc:616] Registered 0 tablets
20:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.530791 19228 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:04:20.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.711880 19228 rpc_server.cc:307] RPC server started. Bound to: 127.18.88.194:44051
20:04:20.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.712036 19358 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.88.194:44051 every 8 connection(s)
20:04:20.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.715286 19228 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-1/data/info.pb
20:04:20.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.725407 18787 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 19228
20:04:20.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.726064 18787 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-1/wal/instance
20:04:20.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.743688 18787 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:20.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:20.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-2/wal
20:04:20.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-2/data
20:04:20.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:20.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:20.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:20.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:20.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:20.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:20.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-2/logs
20:04:20.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-2/data/info.pb
20:04:20.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:20.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:20.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:20.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:20.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:20.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.88.195:0
20:04:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.88.195
20:04:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.88.195
20:04:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.88.254:45489,127.18.88.253:46251,127.18.88.252:41909
20:04:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.88.212:43575
20:04:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:20.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:04:20.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.784883 19360 heartbeater.cc:344] Connected to a master server at 127.18.88.253:46251
20:04:20.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.785385 19360 heartbeater.cc:461] Registering TS with master...
20:04:20.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.786545 19360 heartbeater.cc:507] Master 127.18.88.253:46251 requested a full tablet report, sending...
20:04:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.788818 19359 heartbeater.cc:344] Connected to a master server at 127.18.88.252:41909
20:04:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.789160 19359 heartbeater.cc:461] Registering TS with master...
20:04:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.790088 19359 heartbeater.cc:507] Master 127.18.88.252:41909 requested a full tablet report, sending...
20:04:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.790234 18907 ts_manager.cc:194] Registered new tserver with Master: c9c779fc78f84f2db3c9f93594294bd9 (127.18.88.194:44051)
20:04:20.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.792151 19361 heartbeater.cc:344] Connected to a master server at 127.18.88.254:45489
20:04:20.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.792501 19361 heartbeater.cc:461] Registering TS with master...
20:04:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.792858 18988 ts_manager.cc:194] Registered new tserver with Master: c9c779fc78f84f2db3c9f93594294bd9 (127.18.88.194:44051)
20:04:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.793437 19361 heartbeater.cc:507] Master 127.18.88.254:45489 requested a full tablet report, sending...
20:04:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.794723 18988 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.18.88.194:42279
20:04:20.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:20.797113 18832 ts_manager.cc:194] Registered new tserver with Master: c9c779fc78f84f2db3c9f93594294bd9 (127.18.88.194:44051)
20:04:21.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:21.135082 19369 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:21.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:21.135911 19369 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:21.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:21.136396 19369 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:21.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:21.174446 19369 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:21.175268 19369 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.18.88.195
20:04:21.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:21.216677 19369 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:21.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:21.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.88.212:43575
20:04:21.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:21.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:21.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-2/data
20:04:21.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-2/wal
20:04:21.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.88.195:0
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-2/data/info.pb
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.88.195
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.88.254:45489,127.18.88.253:46251,127.18.88.252:41909
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.88.195
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-2/logs
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:04:21.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:04:21.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:21.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:21.218245 19369 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:21.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:21.219990 19369 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:21.234974 19375 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:21.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:21.799240 19359 heartbeater.cc:499] Master 127.18.88.252:41909 was elected leader, sending a full tablet report...
20:04:22.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:21.236151 19376 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:22.605521 19378 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:22.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:22.609571 19377 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1366 milliseconds
20:04:22.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.609732 19369 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:22.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.611081 19369 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:22.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.614106 19369 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:22.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.615669 19369 hybrid_clock.cc:648] HybridClock initialized: now 1781813062615595 us; error 95 us; skew 500 ppm
20:04:22.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.617048 19369 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:22.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.624392 19369 webserver.cc:533] Webserver started at http://127.18.88.195:39169/ using document root <none> and password file <none>
20:04:22.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.625725 19369 fs_manager.cc:362] Metadata directory not provided
20:04:22.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.626096 19369 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:22.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.626732 19369 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:22.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.631657 19369 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-2/data/instance:
20:04:22.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "55f9cb8345d2459b9cb4dd3dd9eae57e"
20:04:22.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:22 on dist-test-slave-k9gl"
20:04:22.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.633034 19369 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-2/wal/instance:
20:04:22.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "55f9cb8345d2459b9cb4dd3dd9eae57e"
20:04:22.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:22 on dist-test-slave-k9gl"
20:04:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.641364 19369 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.002s sys 0.005s
20:04:22.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.648723 19387 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:22.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.650408 19369 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
20:04:22.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.650830 19369 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-2/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-2/wal
20:04:22.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "55f9cb8345d2459b9cb4dd3dd9eae57e"
20:04:22.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:22 on dist-test-slave-k9gl"
20:04:22.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.651222 19369 fs_report.cc:389] FS layout report
20:04:22.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:22.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-2/wal
20:04:22.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-2/wal
20:04:22.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-2/data/data
20:04:22.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:22.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:22.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:22.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:22.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:22.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:22.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:22.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:22.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:22.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:22.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:22.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.700421 19369 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:22.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.702068 19369 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:22.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.702633 19369 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:22.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.705582 19369 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.710934 19369 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.711201 19369 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.711496 19369 ts_tablet_manager.cc:616] Registered 0 tablets
20:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.711721 19369 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:04:22.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.868371 19369 rpc_server.cc:307] RPC server started. Bound to: 127.18.88.195:46273
20:04:22.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.868551 19499 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.88.195:46273 every 8 connection(s)
20:04:22.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.871124 19369 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-2/data/info.pb
20:04:22.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.881377 18787 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 19369
20:04:22.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.882006 18787 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster10020379584187548464/ts-2/wal/instance
20:04:22.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.945593 19500 heartbeater.cc:344] Connected to a master server at 127.18.88.252:41909
20:04:22.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.946242 19500 heartbeater.cc:461] Registering TS with master...
20:04:22.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.946396 19502 heartbeater.cc:344] Connected to a master server at 127.18.88.254:45489
20:04:22.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.946811 19502 heartbeater.cc:461] Registering TS with master...
20:04:22.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.947362 19501 heartbeater.cc:344] Connected to a master server at 127.18.88.253:46251
20:04:22.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.947777 19501 heartbeater.cc:461] Registering TS with master...
20:04:22.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.947989 19500 heartbeater.cc:507] Master 127.18.88.252:41909 requested a full tablet report, sending...
20:04:22.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.948058 19502 heartbeater.cc:507] Master 127.18.88.254:45489 requested a full tablet report, sending...
20:04:22.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.948773 19501 heartbeater.cc:507] Master 127.18.88.253:46251 requested a full tablet report, sending...
20:04:22.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.951436 18988 ts_manager.cc:194] Registered new tserver with Master: 55f9cb8345d2459b9cb4dd3dd9eae57e (127.18.88.195:46273)
20:04:22.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.951882 18907 ts_manager.cc:194] Registered new tserver with Master: 55f9cb8345d2459b9cb4dd3dd9eae57e (127.18.88.195:46273)
20:04:22.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.952008 18832 ts_manager.cc:194] Registered new tserver with Master: 55f9cb8345d2459b9cb4dd3dd9eae57e (127.18.88.195:46273)
20:04:22.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.953368 18988 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.18.88.195:60129
20:04:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:22.969513 18787 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
20:04:22.982 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:04:22.984 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:04:22.995 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.18.88.254"
port: 45489
}
bound_http_address {
host: "127.18.88.254"
port: 35873
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.18.88.253"
port: 46251
}
bound_http_address {
host: "127.18.88.253"
port: 43615
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.18.88.252"
port: 41909
}
bound_http_address {
host: "127.18.88.252"
port: 36341
}
}
}
20:04:23.000 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:04:23.004 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.18.88.193"
port: 36727
}
bound_http_address {
host: "127.18.88.193"
port: 45925
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.18.88.194"
port: 44051
}
bound_http_address {
host: "127.18.88.194"
port: 46667
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.18.88.195"
port: 46273
}
bound_http_address {
host: "127.18.88.195"
port: 39169
}
}
}
20:04:23.005 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:04:23.197 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
20:04:23.265 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:04:23.390 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task7Ze3A5/test-tmp
20:04:23.391 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:04:23.391 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task7Ze3A5/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:04:23.398 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
20:04:23.825 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:23.825 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:04:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:23.829945 19512 mini_oidc.cc:150] Starting JWKS server
20:04:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:23.834026 19512 webserver.cc:533] Webserver started at http://127.0.0.1:43011/ using document root <none> and password file <none>
20:04:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:23.834530 19512 mini_oidc.cc:186] Starting OIDC Discovery server
20:04:23.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:23.836483 19512 webserver.cc:533] Webserver started at http://127.0.0.1:36315/ using document root <none> and password file <none>
20:04:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
20:04:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/principal' for realm 'KRBTEST.COM',
20:04:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
20:04:23.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:23 dist-test-slave-k9gl krb5kdc[19524](info): setting up network...
20:04:23.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
20:04:23.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:23 dist-test-slave-k9gl krb5kdc[19524](info): set up 2 sockets
20:04:23.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:23 dist-test-slave-k9gl krb5kdc[19524](info): commencing operation
20:04:23.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
20:04:23.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:23.958189 19500 heartbeater.cc:499] Master 127.18.88.252:41909 was elected leader, sending a full tablet report...
20:04:26.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:26.025481 19496 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6fa0 after lost signal to thread 19370
20:04:26.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:26.026651 19496 debug-util.cc:398] Leaking SignalData structure 0x7b08000add40 after lost signal to thread 19499
20:04:26.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:26.090471 19512 mini_kdc.cc:121] Time spent starting KDC: real 2.210s user 0.010s sys 0.017s
20:04:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
20:04:26.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:04:26.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
20:04:26.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
20:04:26.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:04:26.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
20:04:26.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
20:04:26.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:04:26.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
20:04:26.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:04:26.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/test-user.keytab.
20:04:26.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/test-user.keytab.
20:04:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:26 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813066, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:04:26.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-18T20:04:26Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:04:26.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-18T20:04:26Z Disabled control of system clock
20:04:26.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.19.14.62@KRBTEST.COM; defaulting to no policy
20:04:26.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:26.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.19.14.62@KRBTEST.COM" created.
20:04:26.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:26.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.14.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:26.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.14.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:26.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.19.14.62@KRBTEST.COM; defaulting to no policy
20:04:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.19.14.62@KRBTEST.COM" created.
20:04:26.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:26.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.14.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:26.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.14.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:26.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:26.391923 19512 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:26.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:26.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:04:26.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data
20:04:26.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:26.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:26.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:26.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:26.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:26.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/logs
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data/info.pb
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.62:38303
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.62
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.62
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:26.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:26.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:26.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:26.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:26.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:26.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:04:26.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:04:26.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:26.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:26.831213 19541 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:26.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:26.832224 19541 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:26.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:26.833257 19541 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:26.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:26.877985 19541 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:04:26.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:26.878368 19541 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:26.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:26.878870 19541 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:26.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:26.879262 19541 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:26.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:26.879629 19541 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:26.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:26.879979 19541 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:26.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:26.880268 19541 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43011/jwks.json
20:04:26.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:26.927984 19541 master_runner.cc:387] Master server non-default flags:
20:04:26.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:26.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:26.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:26.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:26.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data
20:04:26.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:04:26.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:04:26.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:26.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:26.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:26.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.62
20:04:26.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.62:38303
20:04:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:04:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data/info.pb
20:04:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.62
20:04:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/logs
20:04:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:04:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:04:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:04:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:26.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:26.930335 19541 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:26.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:26.932160 19541 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:26.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:26.949450 19548 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:28.349479 19547 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 19541
20:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:28.939077 19547 kernel_stack_watchdog.cc:198] Thread 19541 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 395ms:
20:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:04:28.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:26.952376 19549 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:28.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:28.941075 19541 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.989s user 0.497s sys 1.113s
20:04:28.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:28.941488 19541 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.990s user 0.497s sys 1.113s
20:04:28.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:28.942423 19550 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1987 milliseconds
20:04:28.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:28.943022 19551 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:28.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:28.942983 19541 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:28.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:28.948578 19541 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:28.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:28.951921 19541 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:28.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:28.953483 19541 hybrid_clock.cc:648] HybridClock initialized: now 1781813068953426 us; error 58 us; skew 500 ppm
20:04:28.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:28.954723 19541 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:28.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:28 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813068, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:28.963577 19541 init.cc:377] Logged in from keytab as kudu/127.19.14.62@KRBTEST.COM (short username kudu)
20:04:28.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:28.971050 19541 webserver.cc:533] Webserver started at http://127.19.14.62:35405/ using document root <none> and password file <none>
20:04:28.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:28.972638 19541 fs_manager.cc:362] Metadata directory not provided
20:04:28.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:28.973131 19541 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:28.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:28.973846 19541 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:28.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:28.980489 19541 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data/instance:
20:04:28.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1acde45546ea4e3196a6946f6fc302c7"
20:04:28.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:28 on dist-test-slave-k9gl"
20:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:28.981907 19541 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal/instance:
20:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1acde45546ea4e3196a6946f6fc302c7"
20:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:28 on dist-test-slave-k9gl"
20:04:28.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:28.992699 19541 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.010s sys 0.000s
20:04:29.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.000320 19562 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:29.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.002264 19541 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.004s sys 0.003s
20:04:29.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.002739 19541 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:04:29.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1acde45546ea4e3196a6946f6fc302c7"
20:04:29.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:28 on dist-test-slave-k9gl"
20:04:29.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.003134 19541 fs_report.cc:389] FS layout report
20:04:29.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:29.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:04:29.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:04:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data/data
20:04:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:29.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.077661 19541 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:29.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.088970 19541 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:29.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.089731 19541 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.187309 19541 rpc_server.cc:307] RPC server started. Bound to: 127.19.14.62:38303
20:04:29.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.187544 19613 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.14.62:38303 every 8 connection(s)
20:04:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.190693 19541 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data/info.pb
20:04:29.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.194159 19512 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 19541
20:04:29.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.194898 19512 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal/instance
20:04:29.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.198297 19614 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:04:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.214388 19614 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } has no permanent_uuid. Determining permanent_uuid...
20:04:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.19.14.61@KRBTEST.COM; defaulting to no policy
20:04:29.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:29.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.19.14.61@KRBTEST.COM" created.
20:04:29.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:29 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813068, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.62@KRBTEST.COM for kudu/127.19.14.62@KRBTEST.COM
20:04:29.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:29.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.14.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:29.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.14.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.270670 19614 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } has no permanent_uuid. Determining permanent_uuid...
20:04:29.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.274070 19564 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:04:29.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.19.14.61@KRBTEST.COM; defaulting to no policy
20:04:29.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.279748 19614 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.61:35907: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:04:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.19.14.61@KRBTEST.COM" created.
20:04:29.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:29.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.14.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:29.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.14.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:29.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.310822 19512 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:29.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:29.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:04:29.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data
20:04:29.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:29.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:29.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/logs
20:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data/info.pb
20:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.61:35907
20:04:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.61
20:04:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.61
20:04:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:29.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:29.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:04:29.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:04:29.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:29.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.329425 19614 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } attempt: 1
20:04:29.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.335755 19614 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.61:35907: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:04:29.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.404357 19614 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } attempt: 2
20:04:29.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.410012 19614 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.61:35907: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:04:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.501447 19614 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } attempt: 3
20:04:29.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.506414 19614 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.61:35907: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:04:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.650002 19614 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } attempt: 4
20:04:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.658443 19614 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.61:35907: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:04:29.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.719554 19623 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:29.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.720329 19623 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:29.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.721194 19623 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:29.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.761301 19623 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:04:29.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.761621 19623 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:29.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.762084 19623 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:29.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.762458 19623 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:29.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.762825 19623 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:29.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.763175 19623 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:29.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.763450 19623 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43011/jwks.json
20:04:29.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.808310 19623 master_runner.cc:387] Master server non-default flags:
20:04:29.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.61
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.61:35907
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data/info.pb
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.61
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:29.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:29.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:29.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:29.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:29.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/logs
20:04:29.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:29.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:29.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:29.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:29.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:29.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:04:29.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:29.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:04:29.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:04:29.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:29.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.810109 19623 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:29.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.811425 19623 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:29.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.823901 19630 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:29.957921 19614 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } attempt: 5
20:04:29.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.966877 19614 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.61:35907: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:04:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:30.514510 19614 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } attempt: 6
20:04:30.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:30.531502 19614 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.61:35907: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:04:31.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.826237 19631 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:31.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:29.831347 19633 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.198782 19623 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:31.199618 19632 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1371 milliseconds
20:04:31.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.203966 19623 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.207294 19623 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:31.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.209025 19623 hybrid_clock.cc:648] HybridClock initialized: now 1781813071208991 us; error 131 us; skew 500 ppm
20:04:31.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.210191 19623 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:31.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:31 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813071, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.219166 19623 init.cc:377] Logged in from keytab as kudu/127.19.14.61@KRBTEST.COM (short username kudu)
20:04:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.225924 19623 webserver.cc:533] Webserver started at http://127.19.14.61:37259/ using document root <none> and password file <none>
20:04:31.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.227020 19623 fs_manager.cc:362] Metadata directory not provided
20:04:31.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.227247 19623 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:31.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.227699 19623 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:31.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.233492 19623 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data/instance:
20:04:31.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "350b6ce39ebc451c8560faceb2efafa7"
20:04:31.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:31 on dist-test-slave-k9gl"
20:04:31.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.234788 19623 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal/instance:
20:04:31.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "350b6ce39ebc451c8560faceb2efafa7"
20:04:31.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:31 on dist-test-slave-k9gl"
20:04:31.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.244547 19623 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.010s sys 0.000s
20:04:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.252657 19644 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:31.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.254572 19623 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.007s sys 0.000s
20:04:31.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.255106 19623 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:04:31.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "350b6ce39ebc451c8560faceb2efafa7"
20:04:31.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:31 on dist-test-slave-k9gl"
20:04:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.255585 19623 fs_report.cc:389] FS layout report
20:04:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:04:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:04:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data/data
20:04:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:31.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:31.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:31.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:31.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.319152 19623 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:31.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.326622 19623 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:31.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.327240 19623 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.417147 19623 rpc_server.cc:307] RPC server started. Bound to: 127.19.14.61:35907
20:04:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.417373 19695 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.14.61:35907 every 8 connection(s)
20:04:31.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.420112 19623 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data/info.pb
20:04:31.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.427486 19696 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:04:31.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.427580 19512 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 19623
20:04:31.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.428068 19512 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal/instance
20:04:31.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.19.14.60@KRBTEST.COM; defaulting to no policy
20:04:31.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.446575 19696 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } has no permanent_uuid. Determining permanent_uuid...
20:04:31.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:31.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.19.14.60@KRBTEST.COM" created.
20:04:31.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:31 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813071, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.61@KRBTEST.COM for kudu/127.19.14.62@KRBTEST.COM
20:04:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.14.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.14.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:31.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.501019 19696 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } has no permanent_uuid. Determining permanent_uuid...
20:04:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.19.14.60@KRBTEST.COM; defaulting to no policy
20:04:31.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:31.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.19.14.60@KRBTEST.COM" created.
20:04:31.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:31 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813071, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.61@KRBTEST.COM for kudu/127.19.14.61@KRBTEST.COM
20:04:31.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.531631 19696 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } has no permanent_uuid. Determining permanent_uuid...
20:04:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:31.535378 19646 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.14.60:46841: connect: Connection refused (error 111)
20:04:31.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:31.540606 19696 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.60:46841: Network error: Client connection negotiation failed: client connection to 127.19.14.60:46841: connect: Connection refused (error 111)
20:04:31.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:31.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.14.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:31.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.14.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:31.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.550657 19512 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:31.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:31.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:04:31.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data
20:04:31.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:31.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:31.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:31.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:31.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:31.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:31.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/logs
20:04:31.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data/info.pb
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.60:46841
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.60
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.60
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:31.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:31.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:04:31.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:04:31.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:31.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.590281 19696 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } attempt: 1
20:04:31.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.592116 19614 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } attempt: 7
20:04:31.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:31.596980 19696 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.60:46841: Network error: Client connection negotiation failed: client connection to 127.19.14.60:46841: connect: Connection refused (error 111)
20:04:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:31 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813068, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.62@KRBTEST.COM for kudu/127.19.14.61@KRBTEST.COM
20:04:31.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.627780 19614 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } has no permanent_uuid. Determining permanent_uuid...
20:04:31.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:31.635743 19614 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.60:46841: Network error: Client connection negotiation failed: client connection to 127.19.14.60:46841: connect: Connection refused (error 111)
20:04:31.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.665566 19696 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } attempt: 2
20:04:31.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:31.671248 19696 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.60:46841: Network error: Client connection negotiation failed: client connection to 127.19.14.60:46841: connect: Connection refused (error 111)
20:04:31.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.694427 19614 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } attempt: 1
20:04:31.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:31.700209 19614 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.60:46841: Network error: Client connection negotiation failed: client connection to 127.19.14.60:46841: connect: Connection refused (error 111)
20:04:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.762768 19696 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } attempt: 3
20:04:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:31.768453 19696 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.60:46841: Network error: Client connection negotiation failed: client connection to 127.19.14.60:46841: connect: Connection refused (error 111)
20:04:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.781831 19614 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } attempt: 2
20:04:31.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:31.787652 19614 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.60:46841: Network error: Client connection negotiation failed: client connection to 127.19.14.60:46841: connect: Connection refused (error 111)
20:04:31.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.873210 19614 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } attempt: 3
20:04:31.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:31.879048 19614 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.60:46841: Network error: Client connection negotiation failed: client connection to 127.19.14.60:46841: connect: Connection refused (error 111)
20:04:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:31.912081 19696 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } attempt: 4
20:04:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:31.918442 19696 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.60:46841: Network error: Client connection negotiation failed: client connection to 127.19.14.60:46841: connect: Connection refused (error 111)
20:04:31.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:31.961529 19706 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:31.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:31.962342 19706 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:31.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:31.963157 19706 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:32.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:32.002619 19706 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:04:32.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:32.003019 19706 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:32.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:32.003509 19706 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:32.003891 19706 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:32.004251 19706 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:32.004596 19706 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:32.004896 19706 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43011/jwks.json
20:04:32.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:32.019520 19614 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } attempt: 4
20:04:32.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:32.024180 19614 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.60:46841: Network error: Client connection negotiation failed: client connection to 127.19.14.60:46841: connect: Connection refused (error 111)
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:32.056306 19706 master_runner.cc:387] Master server non-default flags:
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.60
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.60:46841
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data/info.pb
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.60
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/logs
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:04:32.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:04:32.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:32.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:32.058598 19706 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:32.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:32.060109 19706 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:32.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:32.074453 19715 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:32.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:32.217995 19696 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } attempt: 5
20:04:32.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:32.228184 19696 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.60:46841: Network error: Client connection negotiation failed: client connection to 127.19.14.60:46841: connect: Connection refused (error 111)
20:04:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:32.307785 19614 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } attempt: 5
20:04:32.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:32.319555 19614 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.60:46841: Network error: Client connection negotiation failed: client connection to 127.19.14.60:46841: connect: Connection refused (error 111)
20:04:32.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:32.775827 19696 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } attempt: 6
20:04:32.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:32.783654 19696 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.60:46841: Network error: Client connection negotiation failed: client connection to 127.19.14.60:46841: connect: Connection refused (error 111)
20:04:32.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:32.872689 19614 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } attempt: 6
20:04:32.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:32.899470 19614 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.14.60:46841: Network error: Client connection negotiation failed: client connection to 127.19.14.60:46841: connect: Connection refused (error 111)
20:04:33.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:32.080407 19718 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:33.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:32.076685 19716 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:33.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:33.497797 19717 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1415 milliseconds
20:04:33.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.498122 19706 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:33.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.499694 19706 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.502631 19706 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:33.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.504148 19706 hybrid_clock.cc:648] HybridClock initialized: now 1781813073504099 us; error 54 us; skew 500 ppm
20:04:33.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.505321 19706 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:33 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813073, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:33.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.512553 19706 init.cc:377] Logged in from keytab as kudu/127.19.14.60@KRBTEST.COM (short username kudu)
20:04:33.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.519147 19706 webserver.cc:533] Webserver started at http://127.19.14.60:45819/ using document root <none> and password file <none>
20:04:33.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.520292 19706 fs_manager.cc:362] Metadata directory not provided
20:04:33.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.520565 19706 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:33.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.521142 19706 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:33.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.525820 19706 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data/instance:
20:04:33.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "91fa1f4bb837465ca07067a55ddc3b95"
20:04:33.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:33 on dist-test-slave-k9gl"
20:04:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.527136 19706 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal/instance:
20:04:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "91fa1f4bb837465ca07067a55ddc3b95"
20:04:33.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:33 on dist-test-slave-k9gl"
20:04:33.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.535660 19706 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.002s sys 0.007s
20:04:33.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.543121 19730 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:33.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.545100 19706 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.006s sys 0.000s
20:04:33.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.546244 19706 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:04:33.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "91fa1f4bb837465ca07067a55ddc3b95"
20:04:33.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:33 on dist-test-slave-k9gl"
20:04:33.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.546829 19706 fs_report.cc:389] FS layout report
20:04:33.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:33.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:04:33.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:04:33.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data/data
20:04:33.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:33.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:33.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:33.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:33.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:33.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:33.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:33.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:33.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:33.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:33.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:33.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.608013 19706 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:33.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.614820 19706 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:33.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.615260 19706 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:33.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.705159 19706 rpc_server.cc:307] RPC server started. Bound to: 127.19.14.60:46841
20:04:33.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.705390 19781 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.14.60:46841 every 8 connection(s)
20:04:33.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.708189 19706 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data/info.pb
20:04:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.711342 19512 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 19706
20:04:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.711977 19512 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal/instance
20:04:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.717558 19782 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:04:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.755750 19782 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } has no permanent_uuid. Determining permanent_uuid...
20:04:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:33 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813066, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.14.62@KRBTEST.COM
20:04:33.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:33 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813073, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.60@KRBTEST.COM for kudu/127.19.14.62@KRBTEST.COM
20:04:33.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.798028 19782 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } has no permanent_uuid. Determining permanent_uuid...
20:04:33.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:33 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813066, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.14.61@KRBTEST.COM
20:04:33.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:33 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813073, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.60@KRBTEST.COM for kudu/127.19.14.61@KRBTEST.COM
20:04:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.825918 19782 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } has no permanent_uuid. Determining permanent_uuid...
20:04:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:33 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813066, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.14.60@KRBTEST.COM
20:04:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:33 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813073, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.60@KRBTEST.COM for kudu/127.19.14.60@KRBTEST.COM
20:04:33.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.844249 19696 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } attempt: 7
20:04:33.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:33 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813071, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.61@KRBTEST.COM for kudu/127.19.14.60@KRBTEST.COM
20:04:33.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.864506 19782 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: Bootstrap starting.
20:04:33.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.872918 19782 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: Neither blocks nor log segments found. Creating new log.
20:04:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.875404 19782 log.cc:826] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: Log is configured to *not* fsync() on all Append() calls
20:04:33.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.886211 19782 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: No bootstrap required, opened a new log
20:04:33.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.892477 19696 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: Bootstrap starting.
20:04:33.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.898466 19696 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: Neither blocks nor log segments found. Creating new log.
20:04:33.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.901248 19696 log.cc:826] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: Log is configured to *not* fsync() on all Append() calls
20:04:33.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.910090 19696 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: No bootstrap required, opened a new log
20:04:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.915436 19782 raft_consensus.cc:359] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.916275 19782 raft_consensus.cc:385] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:04:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.916503 19782 raft_consensus.cc:740] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 91fa1f4bb837465ca07067a55ddc3b95, State: Initialized, Role: FOLLOWER
20:04:33.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.917708 19782 consensus_queue.cc:260] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:33.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.921765 19796 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:33.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.922796 19796 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: This master's current role is: FOLLOWER
20:04:33.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.925029 19782 sys_catalog.cc:565] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: configured and running, proceeding with master startup.
20:04:33.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.933369 19614 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } attempt: 7
20:04:33.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.933372 19696 raft_consensus.cc:359] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:33.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.934237 19696 raft_consensus.cc:385] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:04:33.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.934571 19696 raft_consensus.cc:740] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 350b6ce39ebc451c8560faceb2efafa7, State: Initialized, Role: FOLLOWER
20:04:33.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.935781 19696 consensus_queue.cc:260] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:33.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.948895 19802 sys_catalog.cc:455] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:33.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.950425 19802 sys_catalog.cc:458] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: This master's current role is: FOLLOWER
20:04:33.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.951967 19696 sys_catalog.cc:565] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: configured and running, proceeding with master startup.
20:04:33.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:33 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813068, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.62@KRBTEST.COM for kudu/127.19.14.60@KRBTEST.COM
20:04:33.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:33.992502 19815 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:04:33.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:33.998133 19815 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:04:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.000392 19614 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Bootstrap starting.
20:04:34.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.017300 19614 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Neither blocks nor log segments found. Creating new log.
20:04:34.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.022398 19614 log.cc:826] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Log is configured to *not* fsync() on all Append() calls
20:04:34.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:33.988969 19706 master_runner.cc:187] Error getting master registration for 127.19.14.62:38303: OK, instance_id { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" instance_seqno: 1781813069161445 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:04:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.026648 19706 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:04:34.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.031538 19614 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: No bootstrap required, opened a new log
20:04:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.062426 19614 raft_consensus.cc:359] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.063735 19614 raft_consensus.cc:385] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:04:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.064291 19614 raft_consensus.cc:740] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1acde45546ea4e3196a6946f6fc302c7, State: Initialized, Role: FOLLOWER
20:04:34.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.065601 19614 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:34.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:34.080855 19822 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:04:34.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:34.081816 19822 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:04:34.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:34.088099 19650 tablet.cc:2406] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:04:34.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.090427 19824 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:34.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.091902 19824 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: This master's current role is: FOLLOWER
20:04:34.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.095199 19614 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: configured and running, proceeding with master startup.
20:04:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.037482 19623 master_runner.cc:187] Error getting master registration for 127.19.14.62:38303: OK, instance_id { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" instance_seqno: 1781813069161445 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:04:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.103312 19623 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:04:34.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:34.104534 19568 tablet.cc:2406] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:04:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:34.121902 19736 tablet.cc:2406] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:04:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.121826 19824 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:04:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.122594 19824 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:34.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.127781 19671 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1acde45546ea4e3196a6946f6fc302c7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "350b6ce39ebc451c8560faceb2efafa7" is_pre_election: true
20:04:34.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.129084 19671 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1acde45546ea4e3196a6946f6fc302c7 in term 0.
20:04:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.131206 19564 leader_election.cc:304] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1acde45546ea4e3196a6946f6fc302c7, 350b6ce39ebc451c8560faceb2efafa7; no voters:
20:04:34.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.135780 19824 leader_election.cc:290] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907), 91fa1f4bb837465ca07067a55ddc3b95 (127.19.14.60:46841)
20:04:34.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.136708 19824 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:04:34.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.137408 19824 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:04:34.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.138195 19824 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 0 FOLLOWER]: Advancing to term 1
20:04:34.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.142978 19757 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1acde45546ea4e3196a6946f6fc302c7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "91fa1f4bb837465ca07067a55ddc3b95" is_pre_election: true
20:04:34.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.144052 19757 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1acde45546ea4e3196a6946f6fc302c7 in term 0.
20:04:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.164891 19824 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:34.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.169073 19671 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1acde45546ea4e3196a6946f6fc302c7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "350b6ce39ebc451c8560faceb2efafa7"
20:04:34.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.169760 19671 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [term 0 FOLLOWER]: Advancing to term 1
20:04:34.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.184715 19671 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1acde45546ea4e3196a6946f6fc302c7 in term 1.
20:04:34.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.198886 19824 leader_election.cc:290] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 1 election: Requested vote from peers 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907), 91fa1f4bb837465ca07067a55ddc3b95 (127.19.14.60:46841)
20:04:34.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.199241 19757 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1acde45546ea4e3196a6946f6fc302c7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "91fa1f4bb837465ca07067a55ddc3b95"
20:04:34.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.200166 19757 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 0 FOLLOWER]: Advancing to term 1
20:04:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.206280 19564 leader_election.cc:304] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1acde45546ea4e3196a6946f6fc302c7, 350b6ce39ebc451c8560faceb2efafa7; no voters:
20:04:34.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.206835 19836 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:04:34.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.207222 19824 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 1 FOLLOWER]: Leader election won for term 1
20:04:34.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.207765 19824 raft_consensus.cc:697] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 1 LEADER]: Becoming Leader. State: Replica: 1acde45546ea4e3196a6946f6fc302c7, State: Running, Role: LEADER
20:04:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.208758 19824 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:34.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.214622 19757 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1acde45546ea4e3196a6946f6fc302c7 in term 1.
20:04:34.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.232717 19832 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1acde45546ea4e3196a6946f6fc302c7. Latest consensus state: current_term: 1 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:34.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.234282 19832 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: This master's current role is: LEADER
20:04:34.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.254980 19838 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
20:04:34.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.313932 19838 catalog_manager.cc:1519] Initializing Kudu cluster ID...
20:04:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.343641 19757 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 1 FOLLOWER]: Refusing update from remote peer 1acde45546ea4e3196a6946f6fc302c7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:04:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.346211 19671 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [term 1 FOLLOWER]: Refusing update from remote peer 1acde45546ea4e3196a6946f6fc302c7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:04:34.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.350325 19832 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:04:34.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.352054 19832 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:04:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.407686 19796 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1acde45546ea4e3196a6946f6fc302c7. Latest consensus state: current_term: 1 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:34.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.408826 19796 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: This master's current role is: FOLLOWER
20:04:34.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.427651 19840 mvcc.cc:204] Tried to move back new op lower bound from 7298306352484155392 to 7298306351968792576. Current Snapshot: MvccSnapshot[applied={T|T < 7298306352484155392}]
20:04:34.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.433662 19842 mvcc.cc:204] Tried to move back new op lower bound from 7298306352484155392 to 7298306351968792576. Current Snapshot: MvccSnapshot[applied={T|T < 7298306352484155392}]
20:04:34.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.439765 19824 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.440469 19824 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: This master's current role is: LEADER
20:04:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.441133 19838 catalog_manager.cc:1382] Generated new cluster ID: 65300b4ca4c54eb8939d73cea6dbaf4d
20:04:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.441603 19838 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
20:04:34.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.453488 19802 sys_catalog.cc:455] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1acde45546ea4e3196a6946f6fc302c7. Latest consensus state: current_term: 1 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:34.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.454207 19802 sys_catalog.cc:458] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: This master's current role is: FOLLOWER
20:04:34.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.455008 19796 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:34.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.455826 19796 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: This master's current role is: FOLLOWER
20:04:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.462802 19832 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:34.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.464687 19832 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: This master's current role is: LEADER
20:04:34.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.465973 19841 mvcc.cc:204] Tried to move back new op lower bound from 7298306352484155392 to 7298306351968792576. Current Snapshot: MvccSnapshot[applied={T|T < 7298306352484155392}]
20:04:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.480767 19802 sys_catalog.cc:455] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.481490 19802 sys_catalog.cc:458] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: This master's current role is: FOLLOWER
20:04:34.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.521708 19838 catalog_manager.cc:1405] Generated new certificate authority record
20:04:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.524613 19838 catalog_manager.cc:1539] Loading token signing keys...
20:04:34.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.579823 19838 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Generated new TSK 0
20:04:34.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.583021 19838 catalog_manager.cc:1549] Initializing in-progress tserver states...
20:04:34.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.19.14.1@KRBTEST.COM; defaulting to no policy
20:04:34.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:34.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.19.14.1@KRBTEST.COM" created.
20:04:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.14.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.14.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:34.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:34.732625 19512 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.19.14.1: real 0.130s user 0.004s sys 0.010s
20:04:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.19.14.1@KRBTEST.COM; defaulting to no policy
20:04:34.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:34.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.19.14.1@KRBTEST.COM" created.
20:04:34.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:34.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.14.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:34.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.14.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:34.830170 19512 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-0/wal
20:04:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-0/data
20:04:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-0/logs
20:04:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-0/data/info.pb
20:04:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.1:0
20:04:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.14.1
20:04:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.1
20:04:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.1
20:04:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:35.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:35.007155 19815 catalog_manager.cc:1294] Loaded cluster ID: 65300b4ca4c54eb8939d73cea6dbaf4d
20:04:35.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:35.008838 19815 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: loading cluster ID for follower catalog manager: success
20:04:35.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:35.035341 19815 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: acquiring CA information for follower catalog manager: success
20:04:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:35.043083 19815 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:04:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:35.089207 19822 catalog_manager.cc:1294] Loaded cluster ID: 65300b4ca4c54eb8939d73cea6dbaf4d
20:04:35.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:35.089530 19822 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: loading cluster ID for follower catalog manager: success
20:04:35.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:35.103329 19822 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: acquiring CA information for follower catalog manager: success
20:04:35.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:35.122246 19822 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:04:35.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:35.494632 19853 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:35.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:35.495525 19853 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:35.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:35.496510 19853 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:35.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:35.560233 19853 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:35.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:35.561846 19853 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.14.1
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:35.639242 19853 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-0/data
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-0/wal
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.1
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.1:0
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-0/data/info.pb
20:04:35.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.1
20:04:35.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:35.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:35.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.14.1
20:04:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-0/logs
20:04:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:04:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:04:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:04:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:35.641153 19853 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:35.642670 19853 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:35.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:35.688088 19860 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:37.073133 19859 debug-util.cc:398] Leaking SignalData structure 0x7b08000370e0 after lost signal to thread 19853
20:04:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:38.638842 19859 kernel_stack_watchdog.cc:198] Thread 19853 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 383ms:
20:04:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:04:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:04:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:04:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:04:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:38.639449 19853 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.951s user 0.833s sys 1.336s
20:04:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:38.639798 19853 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.952s user 0.833s sys 1.336s
20:04:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:35.690711 19861 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:38.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.670853 19853 server_base.cc:1061] running on GCE node
20:04:38.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:38.672905 19868 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:38.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.674491 19853 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:38.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.689395 19853 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:38.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.690927 19853 hybrid_clock.cc:648] HybridClock initialized: now 1781813078690850 us; error 88 us; skew 500 ppm
20:04:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.692382 19853 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:38.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:38 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813078, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:38.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.701572 19853 init.cc:377] Logged in from keytab as kudu/127.19.14.1@KRBTEST.COM (short username kudu)
20:04:38.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.758481 19853 webserver.cc:533] Webserver started at http://127.19.14.1:41169/ using document root <none> and password file <none>
20:04:38.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.760126 19853 fs_manager.cc:362] Metadata directory not provided
20:04:38.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.760712 19853 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:38.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.761726 19853 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.768844 19853 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-0/data/instance:
20:04:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6509385f6f204a549d529840cc27b524"
20:04:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:38 on dist-test-slave-k9gl"
20:04:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.771052 19853 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-0/wal/instance:
20:04:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6509385f6f204a549d529840cc27b524"
20:04:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:38 on dist-test-slave-k9gl"
20:04:38.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.787894 19853 fs_manager.cc:696] Time spent creating directory manager: real 0.016s user 0.011s sys 0.003s
20:04:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.801900 19874 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:38.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.804622 19853 fs_manager.cc:730] Time spent opening block manager: real 0.012s user 0.002s sys 0.008s
20:04:38.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.805524 19853 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-0/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-0/wal
20:04:38.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6509385f6f204a549d529840cc27b524"
20:04:38.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:38 on dist-test-slave-k9gl"
20:04:38.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.807029 19853 fs_report.cc:389] FS layout report
20:04:38.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:38.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-0/wal
20:04:38.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-0/wal
20:04:38.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-0/data/data
20:04:38.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:38.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.913125 19853 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:38.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.934279 19853 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:38.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.935554 19853 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:38.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.939945 19853 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:38.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.946394 19853 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:04:38.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.946753 19853 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:04:38.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.947093 19853 ts_tablet_manager.cc:616] Registered 0 tablets
20:04:38.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.947341 19853 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:04:38.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:38 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813078, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:38.969966 19557 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:39.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.343019 19853 rpc_server.cc:307] RPC server started. Bound to: 127.19.14.1:46821
20:04:39.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.343150 19987 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.14.1:46821 every 8 connection(s)
20:04:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.346498 19853 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-0/data/info.pb
20:04:39.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.348483 19512 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 19853
20:04:39.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.348985 19512 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-0/wal/instance
20:04:39.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.19.14.2@KRBTEST.COM; defaulting to no policy
20:04:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:39 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813078, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.1@KRBTEST.COM for kudu/127.19.14.61@KRBTEST.COM
20:04:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.19.14.2@KRBTEST.COM" created.
20:04:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:39 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813078, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.1@KRBTEST.COM for kudu/127.19.14.62@KRBTEST.COM
20:04:39.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.436065 19992 heartbeater.cc:344] Connected to a master server at 127.19.14.62:38303
20:04:39.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.436770 19992 heartbeater.cc:461] Registering TS with master...
20:04:39.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:39 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813078, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.1@KRBTEST.COM for kudu/127.19.14.60@KRBTEST.COM
20:04:39.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.438498 19992 heartbeater.cc:507] Master 127.19.14.62:38303 requested a full tablet report, sending...
20:04:39.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.442417 19579 ts_manager.cc:194] Registered new tserver with Master: 6509385f6f204a549d529840cc27b524 (127.19.14.1:46821)
20:04:39.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.445833 19579 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.19.14.1@KRBTEST.COM'} at 127.19.14.1:49607
20:04:39.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.449494 19988 heartbeater.cc:344] Connected to a master server at 127.19.14.61:35907
20:04:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.449932 19988 heartbeater.cc:461] Registering TS with master...
20:04:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.450673 19988 heartbeater.cc:507] Master 127.19.14.61:35907 requested a full tablet report, sending...
20:04:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.454440 19661 ts_manager.cc:194] Registered new tserver with Master: 6509385f6f204a549d529840cc27b524 (127.19.14.1:46821)
20:04:39.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.456943 19989 heartbeater.cc:344] Connected to a master server at 127.19.14.60:46841
20:04:39.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.457338 19989 heartbeater.cc:461] Registering TS with master...
20:04:39.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.458283 19989 heartbeater.cc:507] Master 127.19.14.60:46841 requested a full tablet report, sending...
20:04:39.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:39.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.14.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:39.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.14.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:39.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.461720 19747 ts_manager.cc:194] Registered new tserver with Master: 6509385f6f204a549d529840cc27b524 (127.19.14.1:46821)
20:04:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.19.14.2@KRBTEST.COM; defaulting to no policy
20:04:39.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:39.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.19.14.2@KRBTEST.COM" created.
20:04:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:39.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.14.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:39.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.14.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.514102 19512 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-1/wal
20:04:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-1/data
20:04:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-1/logs
20:04:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-1/data/info.pb
20:04:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:39.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.2:0
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.14.2
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.2
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.2
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:39.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:39.867435 20002 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:39.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:39.868091 20002 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:39.869057 20002 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:39.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:39.912449 20002 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:39.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:39.914157 20002 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.14.2
20:04:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.953895 20002 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-1/data
20:04:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-1/wal
20:04:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.2
20:04:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.2:0
20:04:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-1/data/info.pb
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.2
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.14.2
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-1/logs
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:04:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:39.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.955590 20002 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:39.956951 20002 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:39.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:39.971312 20010 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:40.451431 19992 heartbeater.cc:499] Master 127.19.14.62:38303 was elected leader, sending a full tablet report...
20:04:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:41 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813081, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.226851 19640 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:41.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:41.376902 20009 debug-util.cc:398] Leaking SignalData structure 0x7b08000370e0 after lost signal to thread 20002
20:04:41.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:41.781741 20009 kernel_stack_watchdog.cc:198] Thread 20002 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
20:04:41.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:04:41.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:04:41.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:04:41.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:04:41.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:39.972044 20011 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:41.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:41.783238 20002 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.812s user 0.580s sys 1.176s
20:04:41.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:41.783906 20002 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.813s user 0.581s sys 1.176s
20:04:41.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:41.786288 20013 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:41.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:41.790912 20012 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1812 milliseconds
20:04:41.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.791006 20002 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.792470 20002 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:41.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.795254 20002 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:41.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.796849 20002 hybrid_clock.cc:648] HybridClock initialized: now 1781813081796828 us; error 64 us; skew 500 ppm
20:04:41.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.797873 20002 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:41.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:41 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813081, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.805305 20002 init.cc:377] Logged in from keytab as kudu/127.19.14.2@KRBTEST.COM (short username kudu)
20:04:41.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.810880 20002 webserver.cc:533] Webserver started at http://127.19.14.2:41287/ using document root <none> and password file <none>
20:04:41.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.811908 20002 fs_manager.cc:362] Metadata directory not provided
20:04:41.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.812109 20002 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:41.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.812534 20002 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.817942 20002 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-1/data/instance:
20:04:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2f68907433b4dce88718ce628e2a595"
20:04:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:41 on dist-test-slave-k9gl"
20:04:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.819269 20002 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-1/wal/instance:
20:04:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2f68907433b4dce88718ce628e2a595"
20:04:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:41 on dist-test-slave-k9gl"
20:04:41.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.827749 20002 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.002s sys 0.007s
20:04:41.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.834192 20024 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:41.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.835950 20002 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
20:04:41.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.836335 20002 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-1/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-1/wal
20:04:41.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2f68907433b4dce88718ce628e2a595"
20:04:41.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:41 on dist-test-slave-k9gl"
20:04:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.836779 20002 fs_report.cc:389] FS layout report
20:04:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-1/wal
20:04:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-1/wal
20:04:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-1/data/data
20:04:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:41.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.897768 20002 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:41.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.904759 20002 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:41.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.905265 20002 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.908716 20002 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:41.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.914238 20002 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:04:41.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.914567 20002 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:04:41.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.914798 20002 ts_tablet_manager.cc:616] Registered 0 tablets
20:04:41.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:41.914945 20002 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:04:42.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.095683 20002 rpc_server.cc:307] RPC server started. Bound to: 127.19.14.2:42427
20:04:42.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.095870 20139 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.14.2:42427 every 8 connection(s)
20:04:42.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.098778 20002 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-1/data/info.pb
20:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.101370 19512 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 20002
20:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.101919 19512 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-1/wal/instance
20:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:42 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813081, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.2@KRBTEST.COM for kudu/127.19.14.62@KRBTEST.COM
20:04:42.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:42 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813081, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.2@KRBTEST.COM for kudu/127.19.14.61@KRBTEST.COM
20:04:42.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.19.14.3@KRBTEST.COM; defaulting to no policy
20:04:42.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:42.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.19.14.3@KRBTEST.COM" created.
20:04:42.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.179001 20142 heartbeater.cc:344] Connected to a master server at 127.19.14.62:38303
20:04:42.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.179651 20142 heartbeater.cc:461] Registering TS with master...
20:04:42.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.181361 20142 heartbeater.cc:507] Master 127.19.14.62:38303 requested a full tablet report, sending...
20:04:42.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.183288 20140 heartbeater.cc:344] Connected to a master server at 127.19.14.61:35907
20:04:42.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.183768 20140 heartbeater.cc:461] Registering TS with master...
20:04:42.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:42 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813081, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.2@KRBTEST.COM for kudu/127.19.14.60@KRBTEST.COM
20:04:42.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.184943 20140 heartbeater.cc:507] Master 127.19.14.61:35907 requested a full tablet report, sending...
20:04:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.188428 19579 ts_manager.cc:194] Registered new tserver with Master: a2f68907433b4dce88718ce628e2a595 (127.19.14.2:42427)
20:04:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.188674 19661 ts_manager.cc:194] Registered new tserver with Master: a2f68907433b4dce88718ce628e2a595 (127.19.14.2:42427)
20:04:42.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.190201 19579 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.19.14.2@KRBTEST.COM'} at 127.19.14.2:55939
20:04:42.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.204924 20141 heartbeater.cc:344] Connected to a master server at 127.19.14.60:46841
20:04:42.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.205371 20141 heartbeater.cc:461] Registering TS with master...
20:04:42.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.206161 20141 heartbeater.cc:507] Master 127.19.14.60:46841 requested a full tablet report, sending...
20:04:42.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.209538 19747 ts_manager.cc:194] Registered new tserver with Master: a2f68907433b4dce88718ce628e2a595 (127.19.14.2:42427)
20:04:42.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:42.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.14.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:42.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.14.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.19.14.3@KRBTEST.COM; defaulting to no policy
20:04:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.19.14.3@KRBTEST.COM" created.
20:04:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:42.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.14.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:42.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.14.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab.
20:04:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.267084 19512 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-2/wal
20:04:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-2/data
20:04:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-2/logs
20:04:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-2/data/info.pb
20:04:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.3:0
20:04:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.14.3
20:04:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.3
20:04:42.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:42.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:42.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:42.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.3
20:04:42.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:42.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:42.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:42.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:42.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:42.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:42.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:42.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:42.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:42.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:42.664103 20155 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:42.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:42.664810 20155 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:42.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:42.665441 20155 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:42.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:42.707635 20155 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:42.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:42.709093 20155 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.14.3
20:04:42.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.765112 20155 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-2/data
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-2/wal
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.3
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.3:0
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-2/data/info.pb
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.3
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.14.3
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-2/logs
20:04:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:04:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:04:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:04:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.766556 20155 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:42.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:42.767827 20155 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:42.782912 20162 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:43.195983 20142 heartbeater.cc:499] Master 127.19.14.62:38303 was elected leader, sending a full tablet report...
20:04:43.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:43 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813083, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:43.520962 19726 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:44.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:44.381402 20155 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.593s user 0.000s sys 0.002s
20:04:44.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:44.382824 20155 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.595s user 0.000s sys 0.003s
20:04:44.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:44.195622 20161 debug-util.cc:398] Leaking SignalData structure 0x7b08000370e0 after lost signal to thread 20155
20:04:44.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:42.785914 20163 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:44.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:44.384225 20164 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1593 milliseconds
20:04:44.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.385191 20155 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:44.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:44.385746 20165 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.390565 20155 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:44.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.393749 20155 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:44.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.395354 20155 hybrid_clock.cc:648] HybridClock initialized: now 1781813084395275 us; error 80 us; skew 500 ppm
20:04:44.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.396332 20155 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:44.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:44 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813084, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:44.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.404213 20155 init.cc:377] Logged in from keytab as kudu/127.19.14.3@KRBTEST.COM (short username kudu)
20:04:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.411116 20155 webserver.cc:533] Webserver started at http://127.19.14.3:42473/ using document root <none> and password file <none>
20:04:44.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.412374 20155 fs_manager.cc:362] Metadata directory not provided
20:04:44.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.412629 20155 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:44.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.413597 20155 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:44.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.419625 20155 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-2/data/instance:
20:04:44.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5eb4b8df01144c9284949cab2ce0950c"
20:04:44.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:44 on dist-test-slave-k9gl"
20:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.421232 20155 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-2/wal/instance:
20:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5eb4b8df01144c9284949cab2ce0950c"
20:04:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:44 on dist-test-slave-k9gl"
20:04:44.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.432339 20155 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.005s sys 0.004s
20:04:44.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.440007 20174 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.441725 20155 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.005s sys 0.001s
20:04:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.442229 20155 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-2/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-2/wal
20:04:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5eb4b8df01144c9284949cab2ce0950c"
20:04:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:44 on dist-test-slave-k9gl"
20:04:44.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.442622 20155 fs_report.cc:389] FS layout report
20:04:44.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:44.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-2/wal
20:04:44.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-2/wal
20:04:44.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-2/data/data
20:04:44.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:44.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:44.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:44.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:44.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:44.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:44.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:44.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:44.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:44.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:44.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:44.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.516873 20155 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:44.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.532994 20155 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.533704 20155 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:44.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.538385 20155 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:44.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.544859 20155 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:04:44.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.545197 20155 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
20:04:44.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.545526 20155 ts_tablet_manager.cc:616] Registered 0 tablets
20:04:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.545745 20155 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:04:44.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.823318 20155 rpc_server.cc:307] RPC server started. Bound to: 127.19.14.3:41975
20:04:44.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.823429 20286 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.14.3:41975 every 8 connection(s)
20:04:44.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.826582 20155 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-2/data/info.pb
20:04:44.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.830246 19512 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 20155
20:04:44.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.831113 19512 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/ts-2/wal/instance
20:04:44.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:44 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813084, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.3@KRBTEST.COM for kudu/127.19.14.62@KRBTEST.COM
20:04:44.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:44 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813084, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.3@KRBTEST.COM for kudu/127.19.14.61@KRBTEST.COM
20:04:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:44 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813084, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.3@KRBTEST.COM for kudu/127.19.14.60@KRBTEST.COM
20:04:44.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.914628 20289 heartbeater.cc:344] Connected to a master server at 127.19.14.62:38303
20:04:44.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.915259 20289 heartbeater.cc:461] Registering TS with master...
20:04:44.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.915273 20287 heartbeater.cc:344] Connected to a master server at 127.19.14.61:35907
20:04:44.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.915793 20287 heartbeater.cc:461] Registering TS with master...
20:04:44.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.917547 20287 heartbeater.cc:507] Master 127.19.14.61:35907 requested a full tablet report, sending...
20:04:44.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.918488 20289 heartbeater.cc:507] Master 127.19.14.62:38303 requested a full tablet report, sending...
20:04:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.921254 19661 ts_manager.cc:194] Registered new tserver with Master: 5eb4b8df01144c9284949cab2ce0950c (127.19.14.3:41975)
20:04:44.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.922264 19579 ts_manager.cc:194] Registered new tserver with Master: 5eb4b8df01144c9284949cab2ce0950c (127.19.14.3:41975)
20:04:44.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.924213 19579 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.19.14.3@KRBTEST.COM'} at 127.19.14.3:41503
20:04:44.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.931748 20288 heartbeater.cc:344] Connected to a master server at 127.19.14.60:46841
20:04:44.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.932109 20288 heartbeater.cc:461] Registering TS with master...
20:04:44.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.932745 20288 heartbeater.cc:507] Master 127.19.14.60:46841 requested a full tablet report, sending...
20:04:44.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.936967 19747 ts_manager.cc:194] Registered new tserver with Master: 5eb4b8df01144c9284949cab2ce0950c (127.19.14.3:41975)
20:04:44.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:44.954674 19512 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
20:04:44.967 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:44.969 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
20:04:44.979 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
20:04:44.980 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
20:04:44.985 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.19.14.62"
port: 38303
}
bound_http_address {
host: "127.19.14.62"
port: 35405
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.19.14.61"
port: 35907
}
bound_http_address {
host: "127.19.14.61"
port: 37259
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.19.14.60"
port: 46841
}
bound_http_address {
host: "127.19.14.60"
port: 45819
}
}
}
20:04:44.986 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:04:44.987 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.19.14.1"
port: 46821
}
bound_http_address {
host: "127.19.14.1"
port: 41169
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.19.14.2"
port: 42427
}
bound_http_address {
host: "127.19.14.2"
port: 41287
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.19.14.3"
port: 41975
}
bound_http_address {
host: "127.19.14.3"
port: 42473
}
}
}
20:04:44.988 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:04:44.989 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:04:45.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:45 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813085, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:45.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:04:45.029 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:45.033 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc
20:04:45.046 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:04:45.046 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc
20:04:45.049 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:04:45.074 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
20:04:45.076 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:04:45.077 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:04:45.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:45 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813085, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:45.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:04:45.123 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:45.123 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc
20:04:45.125 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:04:45.126 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.14.62:38303
20:04:45.128 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:04:45.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:45.128921 19512 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 19541
20:04:45.181 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:45.182 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.14.60:46841
20:04:45.182 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:04:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:45.182940 19512 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 19706
20:04:45.221 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:45.221 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.14.61:35907
20:04:45.222 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:04:45.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:45.222445 19512 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 19623
20:04:45.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:45.229823 20142 heartbeater.cc:646] Failed to heartbeat to 127.19.14.62:38303 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.19.14.62:38303: connect: Connection refused (error 111)
20:04:45.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:45.256758 20026 connection.cc:573] client connection to 127.19.14.61:35907 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:04:45.258 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:45.258 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.14.62:38303
20:04:45.262 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:04:45.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:45.264596 19512 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/logs
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data/info.pb
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:45.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:45.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:45.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.62:38303
20:04:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.62
20:04:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35405
20:04:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.62
20:04:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:45.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:04:45.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:04:45.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:45.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:45.495756 19992 heartbeater.cc:646] Failed to heartbeat to 127.19.14.62:38303 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.19.14.62:38303: connect: Connection refused (error 111)
20:04:45.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:45.644528 20303 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:45.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:45.645416 20303 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:45.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:45.646339 20303 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:45.683562 20303 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:04:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:45.683889 20303 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:45.684384 20303 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:45.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:45.684769 20303 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:45.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:45.685158 20303 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:45.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:45.685497 20303 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:45.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:45.685740 20303 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43011/jwks.json
20:04:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:45.727196 20303 master_runner.cc:387] Master server non-default flags:
20:04:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.62
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.62:38303
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data/info.pb
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.62
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35405
20:04:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:45.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:45.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:45.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/logs
20:04:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:04:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:04:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:04:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:45.728957 20303 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:45.730334 20303 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:45.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:45.930035 20289 heartbeater.cc:499] Master 127.19.14.62:38303 was elected leader, sending a full tablet report...
20:04:45.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:45.947064 20289 heartbeater.cc:646] Failed to heartbeat to 127.19.14.62:38303 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.19.14.62:38303: connect: Connection refused (error 111)
20:04:47.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:45.744024 20312 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:47.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:47.146914 20311 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 20303
20:04:47.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:45.743985 20313 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:47.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:47.797247 20303 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.053s user 0.715s sys 1.155s
20:04:47.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:47.798496 20303 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.054s user 0.715s sys 1.155s
20:04:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:47.802713 20315 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:47.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:47.805802 20314 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2057 milliseconds
20:04:47.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:47.805925 20303 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:47.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:47.807756 20303 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:47.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:47.810925 20303 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:47.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:47.812343 20303 hybrid_clock.cc:648] HybridClock initialized: now 1781813087812270 us; error 111 us; skew 500 ppm
20:04:47.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:47.813329 20303 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:47.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:47 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813087, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:47.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:47.820348 20303 init.cc:377] Logged in from keytab as kudu/127.19.14.62@KRBTEST.COM (short username kudu)
20:04:47.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:47.826357 20303 webserver.cc:533] Webserver started at http://127.19.14.62:35405/ using document root <none> and password file <none>
20:04:47.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:47.827370 20303 fs_manager.cc:362] Metadata directory not provided
20:04:47.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:47.827651 20303 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:47.838979 20303 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.007s sys 0.000s
20:04:47.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:47.844300 20339 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:47.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:47.846088 20303 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.002s
20:04:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:47.846704 20303 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:04:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1acde45546ea4e3196a6946f6fc302c7"
20:04:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:28 on dist-test-slave-k9gl"
20:04:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:47.848946 20303 fs_report.cc:389] FS layout report
20:04:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:04:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:04:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data/data
20:04:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:47.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:47.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:47.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:47.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:47.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:47.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:47.907985 20303 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:47.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:47.914731 20303 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:47.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:47.915350 20303 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:48.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.016775 20303 rpc_server.cc:307] RPC server started. Bound to: 127.19.14.62:38303
20:04:48.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.017352 20392 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.14.62:38303 every 8 connection(s)
20:04:48.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.021142 20303 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data/info.pb
20:04:48.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.026047 19512 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 20303
20:04:48.027 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:48.027 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.14.60:46841
20:04:48.027 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:04:48.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.029716 19512 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:48.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:48.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:04:48.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data
20:04:48.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:48.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:48.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:48.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:48.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:48.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:48.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/logs
20:04:48.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data/info.pb
20:04:48.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:48.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:48.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:48.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:48.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:48.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:48.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:48.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:48.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:48.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:48.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:48.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:48.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.60:46841
20:04:48.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.60
20:04:48.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45819
20:04:48.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:48.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.60
20:04:48.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:48.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:48.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:48.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:48.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:48.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:48.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:48.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:48.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:48.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:48.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:48.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:04:48.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:04:48.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:48.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.036124 20394 sys_catalog.cc:263] Verifying existing consensus state
20:04:48.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.045230 20394 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Bootstrap starting.
20:04:48.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.096472 20394 log.cc:826] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Log is configured to *not* fsync() on all Append() calls
20:04:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.118220 20394 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:48.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.119052 20394 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Bootstrap complete.
20:04:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.146855 20394 raft_consensus.cc:359] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.150400 20394 raft_consensus.cc:740] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1acde45546ea4e3196a6946f6fc302c7, State: Initialized, Role: FOLLOWER
20:04:48.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.152570 20394 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:48.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.157724 20398 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.162120 20398 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: This master's current role is: FOLLOWER
20:04:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.164491 20394 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: configured and running, proceeding with master startup.
20:04:48.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.216084 20409 catalog_manager.cc:1294] Loaded cluster ID: 65300b4ca4c54eb8939d73cea6dbaf4d
20:04:48.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.216414 20409 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: loading cluster ID for follower catalog manager: success
20:04:48.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.225477 20409 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: acquiring CA information for follower catalog manager: success
20:04:48.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:48 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813087, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.62@KRBTEST.COM for kudu/127.19.14.62@KRBTEST.COM
20:04:48.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.230559 20409 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:04:48.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.250432 20303 master_runner.cc:187] Error getting master registration for 127.19.14.61:35907: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111),
20:04:48.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.295615 20303 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:04:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.342808 20142 heartbeater.cc:344] Connected to a master server at 127.19.14.62:38303
20:04:48.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.350226 20356 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a2f68907433b4dce88718ce628e2a595" instance_seqno: 1781813082057972) as {username='kudu', principal='kudu/127.19.14.2@KRBTEST.COM'} at 127.19.14.2:45633; Asking this server to re-register.
20:04:48.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.352298 20142 heartbeater.cc:461] Registering TS with master...
20:04:48.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.357246 20355 ts_manager.cc:194] Registered new tserver with Master: a2f68907433b4dce88718ce628e2a595 (127.19.14.2:42427)
20:04:48.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.559605 19992 heartbeater.cc:344] Connected to a master server at 127.19.14.62:38303
20:04:48.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.563324 20356 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "6509385f6f204a549d529840cc27b524" instance_seqno: 1781813079281773) as {username='kudu', principal='kudu/127.19.14.1@KRBTEST.COM'} at 127.19.14.1:47663; Asking this server to re-register.
20:04:48.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:48.563628 20396 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:48.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:48.564611 20396 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:48.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.565505 19992 heartbeater.cc:461] Registering TS with master...
20:04:48.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:48.565629 20396 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:48.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.569451 20356 ts_manager.cc:194] Registered new tserver with Master: 6509385f6f204a549d529840cc27b524 (127.19.14.1:46821)
20:04:48.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:48.615697 20396 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:04:48.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:48.616051 20396 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:48.616706 20396 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:48.617246 20396 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:48.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:48.617655 20396 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:48.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:48.618069 20396 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:48.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:48.618353 20396 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43011/jwks.json
20:04:48.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.677909 20396 master_runner.cc:387] Master server non-default flags:
20:04:48.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:48.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:48.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.60
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.60:46841
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:48.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:48.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:48.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data/info.pb
20:04:48.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.60
20:04:48.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45819
20:04:48.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:48.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:48.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:48.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:48.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:48.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/logs
20:04:48.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:48.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:48.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:48.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:48.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:48.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:04:48.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:48.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:04:48.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:04:48.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:48.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.681252 20396 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:48.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.683123 20396 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:48.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:48.710464 20424 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:48.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:48 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813088, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:48.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:48.743149 19870 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:49.060428 20289 heartbeater.cc:344] Connected to a master server at 127.19.14.62:38303
20:04:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:49.061518 20289 heartbeater.cc:499] Master 127.19.14.62:38303 was elected leader, sending a full tablet report...
20:04:49.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:49.066056 20356 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "5eb4b8df01144c9284949cab2ce0950c" instance_seqno: 1781813084768865) as {username='kudu', principal='kudu/127.19.14.3@KRBTEST.COM'} at 127.19.14.3:52993; Asking this server to re-register.
20:04:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:49.069530 20289 heartbeater.cc:461] Registering TS with master...
20:04:49.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:49.075487 20356 ts_manager.cc:194] Registered new tserver with Master: 5eb4b8df01144c9284949cab2ce0950c (127.19.14.3:41975)
20:04:49.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:49.298228 20432 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:04:49.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:49.299700 20432 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:49.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:49.319169 20432 leader_election.cc:290] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907), 91fa1f4bb837465ca07067a55ddc3b95 (127.19.14.60:46841)
20:04:49.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:49.330506 20341 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:04:49.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:49.345986 20341 leader_election.cc:336] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907): Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:04:49.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:49.353094 20341 leader_election.cc:336] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 91fa1f4bb837465ca07067a55ddc3b95 (127.19.14.60:46841): Network error: Client connection negotiation failed: client connection to 127.19.14.60:46841: connect: Connection refused (error 111)
20:04:49.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:49.359385 20341 leader_election.cc:304] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1acde45546ea4e3196a6946f6fc302c7; no voters: 350b6ce39ebc451c8560faceb2efafa7, 91fa1f4bb837465ca07067a55ddc3b95
20:04:49.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:49.361390 20432 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
20:04:50.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:48.730281 20425 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:50.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:50.104903 20423 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 20396
20:04:50.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:50.546299 20396 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.843s user 0.670s sys 0.996s
20:04:50.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:50.547430 20396 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.845s user 0.670s sys 0.997s
20:04:50.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.559183 20396 server_base.cc:1061] running on GCE node
20:04:50.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:50.559841 20444 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:50.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.561606 20396 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:50.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.565266 20396 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:50.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.566777 20396 hybrid_clock.cc:648] HybridClock initialized: now 1781813090566698 us; error 86 us; skew 500 ppm
20:04:50.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.568150 20396 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:50.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:50 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813090, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:50.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.577267 20396 init.cc:377] Logged in from keytab as kudu/127.19.14.60@KRBTEST.COM (short username kudu)
20:04:50.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.584720 20396 webserver.cc:533] Webserver started at http://127.19.14.60:45819/ using document root <none> and password file <none>
20:04:50.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.586073 20396 fs_manager.cc:362] Metadata directory not provided
20:04:50.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.586445 20396 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:50.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.599190 20396 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.004s sys 0.004s
20:04:50.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.606546 20452 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:50.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.608414 20396 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.002s sys 0.003s
20:04:50.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.609192 20396 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:04:50.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "91fa1f4bb837465ca07067a55ddc3b95"
20:04:50.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:33 on dist-test-slave-k9gl"
20:04:50.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.612473 20396 fs_report.cc:389] FS layout report
20:04:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:04:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:04:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data/data
20:04:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:50.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.709192 20396 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:50.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.717962 20396 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:50.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.718637 20396 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:50.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.844238 20396 rpc_server.cc:307] RPC server started. Bound to: 127.19.14.60:46841
20:04:50.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.844305 20503 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.14.60:46841 every 8 connection(s)
20:04:50.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.847692 20396 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data/info.pb
20:04:50.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.851454 19512 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 20396
20:04:50.853 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:50.854 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.14.61:35907
20:04:50.854 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:04:50.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.857100 19512 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:50.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:50.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:04:50.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data
20:04:50.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:50.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:50.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:50.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:50.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:50.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/logs
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data/info.pb
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.61:35907
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.61
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37259
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.61
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:04:50.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:04:50.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:50.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.865373 20504 sys_catalog.cc:263] Verifying existing consensus state
20:04:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.876231 20504 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: Bootstrap starting.
20:04:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.933910 20504 log.cc:826] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: Log is configured to *not* fsync() on all Append() calls
20:04:50.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.959213 20504 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:50.960412 20504 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: Bootstrap complete.
20:04:51.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.003799 20504 raft_consensus.cc:359] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:51.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.008047 20504 raft_consensus.cc:740] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 91fa1f4bb837465ca07067a55ddc3b95, State: Initialized, Role: FOLLOWER
20:04:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.009719 20504 consensus_queue.cc:260] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:51.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.031177 20508 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:51.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.032891 20508 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: This master's current role is: FOLLOWER
20:04:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.033149 20504 sys_catalog.cc:565] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: configured and running, proceeding with master startup.
20:04:51.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.053542 20510 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:04:51.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.054070 20510 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:51.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.069300 20510 leader_election.cc:290] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907), 91fa1f4bb837465ca07067a55ddc3b95 (127.19.14.60:46841)
20:04:51.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:51.127705 20341 leader_election.cc:336] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907): Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:04:51.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:51 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813087, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.62@KRBTEST.COM for kudu/127.19.14.60@KRBTEST.COM
20:04:51.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.178325 20479 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1acde45546ea4e3196a6946f6fc302c7" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "91fa1f4bb837465ca07067a55ddc3b95" is_pre_election: true
20:04:51.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.179577 20479 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1acde45546ea4e3196a6946f6fc302c7 in term 1.
20:04:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.183548 20341 leader_election.cc:304] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1acde45546ea4e3196a6946f6fc302c7, 91fa1f4bb837465ca07067a55ddc3b95; no voters: 350b6ce39ebc451c8560faceb2efafa7
20:04:51.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.184859 20288 heartbeater.cc:344] Connected to a master server at 127.19.14.60:46841
20:04:51.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.184986 20510 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 1 FOLLOWER]: Leader pre-election won for term 2
20:04:51.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.185331 20510 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:04:51.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.185642 20510 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 1 FOLLOWER]: Advancing to term 2
20:04:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.196259 20510 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:51.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.200978 20510 leader_election.cc:290] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 2 election: Requested vote from peers 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907), 91fa1f4bb837465ca07067a55ddc3b95 (127.19.14.60:46841)
20:04:51.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.202853 20479 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1acde45546ea4e3196a6946f6fc302c7" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "91fa1f4bb837465ca07067a55ddc3b95"
20:04:51.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.206552 20527 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:04:51.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.209875 20479 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 1 FOLLOWER]: Advancing to term 2
20:04:51.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:51.212792 20341 leader_election.cc:336] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907): Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:04:51.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.226330 20479 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1acde45546ea4e3196a6946f6fc302c7 in term 2.
20:04:51.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.232045 20341 leader_election.cc:304] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1acde45546ea4e3196a6946f6fc302c7, 91fa1f4bb837465ca07067a55ddc3b95; no voters: 350b6ce39ebc451c8560faceb2efafa7
20:04:51.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.232894 20510 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 2 FOLLOWER]: Leader election won for term 2
20:04:51.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.239720 20510 raft_consensus.cc:697] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 2 LEADER]: Becoming Leader. State: Replica: 1acde45546ea4e3196a6946f6fc302c7, State: Running, Role: LEADER
20:04:51.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.241102 20510 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:51.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:51 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813090, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.60@KRBTEST.COM for kudu/127.19.14.62@KRBTEST.COM
20:04:51.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.270188 20530 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1acde45546ea4e3196a6946f6fc302c7. Latest consensus state: current_term: 2 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:51.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.273631 20530 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: This master's current role is: LEADER
20:04:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.282795 20533 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
20:04:51.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.306481 20533 catalog_manager.cc:1519] Initializing Kudu cluster ID...
20:04:51.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.320284 20533 catalog_manager.cc:1294] Loaded cluster ID: 65300b4ca4c54eb8939d73cea6dbaf4d
20:04:51.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.324923 20533 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
20:04:51.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.334223 20533 catalog_manager.cc:1539] Loading token signing keys...
20:04:51.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.347848 20533 catalog_manager.cc:6080] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Loaded TSK: 0
20:04:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.314123 20396 master_runner.cc:187] Error getting master registration for 127.19.14.61:35907: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111),
20:04:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.350270 20396 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:04:51.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.351063 20533 catalog_manager.cc:1549] Initializing in-progress tserver states...
20:04:51.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:51.417606 20518 negotiation.cc:336] Failed RPC negotiation. Trace:
20:04:51.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:04:51.408231 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.19.14.2:56943 (local address 127.19.14.60:46841)
20:04:51.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:04:51.409245 (+ 1014us) server_negotiation.cc:207] Beginning negotiation
20:04:51.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:04:51.409286 (+ 41us) server_negotiation.cc:400] Waiting for connection header
20:04:51.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:04:51.410671 (+ 1385us) server_negotiation.cc:408] Connection header received
20:04:51.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:04:51.411192 (+ 521us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:04:51.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:04:51.411213 (+ 21us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:04:51.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:04:51.411486 (+ 273us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:04:51.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:04:51.411655 (+ 169us) server_negotiation.cc:227] Negotiated authn=SASL
20:04:51.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:04:51.415367 (+ 3712us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.19.14.2:56943: BlockingRecv error: recv got EOF from 127.19.14.2:56943 (error 108)
20:04:51.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":140}
20:04:51.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:51.446787 20506 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:51.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:51.447674 20506 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:51.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:51.448925 20506 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:51.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:51.489319 20506 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:04:51.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:51.489676 20506 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:51.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:51.490511 20506 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:51.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:51.490902 20506 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:51.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:51.491267 20506 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:51.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:51.491616 20506 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:51.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:51.491876 20506 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43011/jwks.json
20:04:51.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.548651 20506 master_runner.cc:387] Master server non-default flags:
20:04:51.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:51.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:51.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:51.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:51.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data
20:04:51.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:04:51.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.61
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.61:35907
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data/info.pb
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.61
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37259
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:51.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:51.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/logs
20:04:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:04:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:04:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:04:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.550660 20506 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.552013 20506 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:51.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:51.569329 20542 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:51.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:51 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813088, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.1@KRBTEST.COM for kudu/127.19.14.60@KRBTEST.COM
20:04:51.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.652490 19989 heartbeater.cc:344] Connected to a master server at 127.19.14.60:46841
20:04:51.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.663184 20469 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "6509385f6f204a549d529840cc27b524" instance_seqno: 1781813079281773) as {username='kudu', principal='kudu/127.19.14.1@KRBTEST.COM'} at 127.19.14.1:52123; Asking this server to re-register.
20:04:51.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.665392 19989 heartbeater.cc:461] Registering TS with master...
20:04:51.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.670961 20469 ts_manager.cc:194] Registered new tserver with Master: 6509385f6f204a549d529840cc27b524 (127.19.14.1:46821)
20:04:51.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:51.765820 20341 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 -> Peer 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907): Couldn't send request to peer 350b6ce39ebc451c8560faceb2efafa7. Status: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:04:51.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.795620 20479 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 2 FOLLOWER]: Refusing update from remote peer 1acde45546ea4e3196a6946f6fc302c7: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
20:04:51.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.797230 20530 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
20:04:51.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:51 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813091, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:51.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.813071 20020 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:51.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.820276 20551 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1acde45546ea4e3196a6946f6fc302c7. Latest consensus state: current_term: 2 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:51.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.823895 20551 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: This master's current role is: FOLLOWER
20:04:51.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.826076 20530 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:51.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.827057 20530 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: This master's current role is: LEADER
20:04:51.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.831166 20551 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:51.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:51.832327 20551 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: This master's current role is: FOLLOWER
20:04:52.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:52.194710 20469 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "5eb4b8df01144c9284949cab2ce0950c" instance_seqno: 1781813084768865) as {username='kudu', principal='kudu/127.19.14.3@KRBTEST.COM'} at 127.19.14.3:40891; Asking this server to re-register.
20:04:52.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:52.197619 20288 heartbeater.cc:461] Registering TS with master...
20:04:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:52.202992 20469 ts_manager.cc:194] Registered new tserver with Master: 5eb4b8df01144c9284949cab2ce0950c (127.19.14.3:41975)
20:04:52.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:52.253782 20527 catalog_manager.cc:1294] Loaded cluster ID: 65300b4ca4c54eb8939d73cea6dbaf4d
20:04:52.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:52.254739 20527 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: loading cluster ID for follower catalog manager: success
20:04:52.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:52.269384 20527 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: acquiring CA information for follower catalog manager: success
20:04:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:52.281670 20527 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:04:52.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:52.378958 20142 heartbeater.cc:499] Master 127.19.14.62:38303 was elected leader, sending a full tablet report...
20:04:52.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:52.479838 20141 heartbeater.cc:344] Connected to a master server at 127.19.14.60:46841
20:04:52.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:52.484117 20469 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a2f68907433b4dce88718ce628e2a595" instance_seqno: 1781813082057972) as {username='kudu', principal='kudu/127.19.14.2@KRBTEST.COM'} at 127.19.14.2:34867; Asking this server to re-register.
20:04:52.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:52.486873 20141 heartbeater.cc:461] Registering TS with master...
20:04:52.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:52.491443 20469 ts_manager.cc:194] Registered new tserver with Master: a2f68907433b4dce88718ce628e2a595 (127.19.14.2:42427)
20:04:52.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:52.588203 19992 heartbeater.cc:499] Master 127.19.14.62:38303 was elected leader, sending a full tablet report...
20:04:53.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.092672 20289 heartbeater.cc:499] Master 127.19.14.62:38303 was elected leader, sending a full tablet report...
20:04:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.304267 20303 master_runner.cc:187] Error getting master registration for 127.19.14.61:35907: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111),
20:04:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.304929 20303 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:04:53.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:51.570369 20541 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:53.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:53.361825 20544 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:53.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:53.368919 20543 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1799 milliseconds
20:04:53.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.369035 20506 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.370879 20506 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:53.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.376133 20506 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.380286 20506 hybrid_clock.cc:648] HybridClock initialized: now 1781813093380235 us; error 63 us; skew 500 ppm
20:04:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.381534 20506 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:53.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:53 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813093, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:53.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.392127 20506 init.cc:377] Logged in from keytab as kudu/127.19.14.61@KRBTEST.COM (short username kudu)
20:04:53.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.400115 20506 webserver.cc:533] Webserver started at http://127.19.14.61:37259/ using document root <none> and password file <none>
20:04:53.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.401733 20506 fs_manager.cc:362] Metadata directory not provided
20:04:53.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.402097 20506 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:53.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.416913 20506 fs_manager.cc:714] Time spent opening directory manager: real 0.009s user 0.006s sys 0.005s
20:04:53.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.425459 20572 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:53.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.427248 20506 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.000s sys 0.003s
20:04:53.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.427791 20506 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:04:53.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "350b6ce39ebc451c8560faceb2efafa7"
20:04:53.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:31 on dist-test-slave-k9gl"
20:04:53.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.431165 20506 fs_report.cc:389] FS layout report
20:04:53.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:53.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:04:53.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:04:53.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data/data
20:04:53.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:53.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:53.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:53.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:53.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:53.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:53.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:53.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:53.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.519183 20506 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:53.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.525717 20506 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:53.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.526162 20506 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:53.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.676398 20506 rpc_server.cc:307] RPC server started. Bound to: 127.19.14.61:35907
20:04:53.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.677543 20625 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.14.61:35907 every 8 connection(s)
20:04:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.680523 20506 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data/info.pb
20:04:53.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.686360 19512 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 20506
20:04:53.688 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:53.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.716351 20626 sys_catalog.cc:263] Verifying existing consensus state
20:04:53.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.728873 20626 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: Bootstrap starting.
20:04:53.791 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.14.62:38303(127.19.14.62:38303)] connecting to peer
20:04:53.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.798393 20626 log.cc:826] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: Log is configured to *not* fsync() on all Append() calls
20:04:53.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.828893 20626 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:53.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.834769 20626 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: Bootstrap complete.
20:04:53.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.883051 20626 raft_consensus.cc:359] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:53.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.888090 20626 raft_consensus.cc:740] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 350b6ce39ebc451c8560faceb2efafa7, State: Initialized, Role: FOLLOWER
20:04:53.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.890363 20626 consensus_queue.cc:260] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:53.905 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.14.60:46841(127.19.14.60:46841)] connecting to peer
20:04:53.916 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.14.61:35907(127.19.14.61:35907)] connecting to peer
20:04:53.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.925927 20631 sys_catalog.cc:455] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.928597 20631 sys_catalog.cc:458] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: This master's current role is: FOLLOWER
20:04:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:53.927834 20626 sys_catalog.cc:565] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: configured and running, proceeding with master startup.
20:04:53.961 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.19.14.61:35907(127.19.14.61:35907)] Successfully connected to peer
20:04:53.965 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.19.14.60:46841(127.19.14.60:46841)] Successfully connected to peer
20:04:53.977 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.19.14.62:38303(127.19.14.62:38303)] Successfully connected to peer
20:04:54.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:54.082082 20648 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:04:54.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:54 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813093, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.61@KRBTEST.COM for kudu/127.19.14.62@KRBTEST.COM
20:04:54.139 [WARN - kudu-nio-0] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
20:04:54.193 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.19.14.60
20:04:54.197 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.19.14.61
20:04:54.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:54 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813093, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.61@KRBTEST.COM for kudu/127.19.14.61@KRBTEST.COM
20:04:54.201 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.19.14.62
20:04:54.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:54.210496 20653 negotiation.cc:336] Failed RPC negotiation. Trace:
20:04:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:04:54.176785 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.19.14.3:59843 (local address 127.19.14.61:35907)
20:04:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:04:54.189498 (+ 12713us) server_negotiation.cc:207] Beginning negotiation
20:04:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:04:54.189535 (+ 37us) server_negotiation.cc:400] Waiting for connection header
20:04:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:04:54.189593 (+ 58us) server_negotiation.cc:408] Connection header received
20:04:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:04:54.202420 (+ 12827us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:04:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:04:54.202465 (+ 45us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:04:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:04:54.202795 (+ 330us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:04:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:04:54.203069 (+ 274us) server_negotiation.cc:227] Negotiated authn=SASL
20:04:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:04:54.208509 (+ 5440us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.19.14.3:59843: BlockingRecv error: recv got EOF from 127.19.14.3:59843 (error 108)
20:04:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":11939,"thread_start_us":23986,"threads_started":1}
20:04:54.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:54 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813093, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.61@KRBTEST.COM for kudu/127.19.14.60@KRBTEST.COM
20:04:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:54 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813087, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.62@KRBTEST.COM for kudu/127.19.14.61@KRBTEST.COM
20:04:54.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:54.320936 20600 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [term 1 FOLLOWER]: Advancing to term 2
20:04:54.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:54.403375 20659 sys_catalog.cc:455] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:54.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:54.404815 20659 sys_catalog.cc:458] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: This master's current role is: FOLLOWER
20:04:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:54 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813094, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:54.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:54.409118 20631 sys_catalog.cc:455] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1acde45546ea4e3196a6946f6fc302c7. Latest consensus state: current_term: 2 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:54.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:54.409981 20631 sys_catalog.cc:458] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: This master's current role is: FOLLOWER
20:04:54.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:54.410193 20170 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:54.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:54 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813091, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.2@KRBTEST.COM for kudu/127.19.14.61@KRBTEST.COM
20:04:54.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:54.536600 20140 heartbeater.cc:344] Connected to a master server at 127.19.14.61:35907
20:04:54.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:54.551518 20590 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a2f68907433b4dce88718ce628e2a595" instance_seqno: 1781813082057972) as {username='kudu', principal='kudu/127.19.14.2@KRBTEST.COM'} at 127.19.14.2:42829; Asking this server to re-register.
20:04:54.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:54.555719 20140 heartbeater.cc:461] Registering TS with master...
20:04:54.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:54.563164 20589 ts_manager.cc:194] Registered new tserver with Master: a2f68907433b4dce88718ce628e2a595 (127.19.14.2:42427)
20:04:54.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:54 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813088, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.1@KRBTEST.COM for kudu/127.19.14.61@KRBTEST.COM
20:04:54.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:54.705528 19988 heartbeater.cc:344] Connected to a master server at 127.19.14.61:35907
20:04:54.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:54.709829 20589 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "6509385f6f204a549d529840cc27b524" instance_seqno: 1781813079281773) as {username='kudu', principal='kudu/127.19.14.1@KRBTEST.COM'} at 127.19.14.1:54535; Asking this server to re-register.
20:04:54.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:54.711431 19988 heartbeater.cc:461] Registering TS with master...
20:04:54.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:54.715083 20589 ts_manager.cc:194] Registered new tserver with Master: 6509385f6f204a549d529840cc27b524 (127.19.14.1:46821)
20:04:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:54.773521 20662 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:54.774325 20662 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: This master's current role is: LEADER
20:04:54.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:54 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1781813085, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.14.60@KRBTEST.COM
20:04:54.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:54 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1781813085, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.14.62@KRBTEST.COM
20:04:54.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:54 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1781813085, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.14.61@KRBTEST.COM
20:04:54.957 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xcef265bc, L:/127.0.0.1:52680 - R:/127.19.14.60:46841] using SASL/GSSAPI
20:04:54.957 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x7608aaf0, L:/127.0.0.1:44972 - R:/127.19.14.62:38303] using SASL/GSSAPI
20:04:54.972 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xd4f5ecc1, L:/127.0.0.1:55026 - R:/127.19.14.61:35907] using SASL/GSSAPI
20:04:55.030 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
20:04:55.032 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1335), Traces: [0ms] refreshing cache from master, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.14.62:38303, [203ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.14.60:46841, [214ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.14.61:35907, [1287ms] Sub RPC ConnectToMaster: received response from server master-127.19.14.62:38303: OK, [1290ms] Sub RPC ConnectToMaster: received response from server master-127.19.14.61:35907: OK, [1290ms] Sub RPC ConnectToMaster: received response from server master-127.19.14.60:46841: OK, deferred=null) after lookup
20:04:55.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:55.113427 20648 catalog_manager.cc:1294] Loaded cluster ID: 65300b4ca4c54eb8939d73cea6dbaf4d
20:04:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:55.114218 20648 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: loading cluster ID for follower catalog manager: success
20:04:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:55.128904 20648 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: acquiring CA information for follower catalog manager: success
20:04:55.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:55.141009 20648 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:04:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:55.247954 20287 heartbeater.cc:344] Connected to a master server at 127.19.14.61:35907
20:04:55.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:55.255345 20589 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "5eb4b8df01144c9284949cab2ce0950c" instance_seqno: 1781813084768865) as {username='kudu', principal='kudu/127.19.14.3@KRBTEST.COM'} at 127.19.14.3:34659; Asking this server to re-register.
20:04:55.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:55.267481 20287 heartbeater.cc:461] Registering TS with master...
20:04:55.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:55.273417 20589 ts_manager.cc:194] Registered new tserver with Master: 5eb4b8df01144c9284949cab2ce0950c (127.19.14.3:41975)
20:04:56.083 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:04:56.084 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:04:56.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:56 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813096, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:56.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:04:56.128 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:56.128 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc
20:04:56.130 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:04:56.131 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.14.62:38303
20:04:56.131 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:04:56.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:56.132085 19512 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 20303
20:04:56.161 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:56.162 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.14.60:46841
20:04:56.162 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:04:56.163 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.19.14.62:38303(127.19.14.62:38303)] handling channelInactive
20:04:56.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.14.62:38303(127.19.14.62:38303)] cleaning up while in state READY due to: connection closed
20:04:56.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:56.168969 19512 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 20396
20:04:56.217 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.19.14.60:46841(127.19.14.60:46841)] handling channelInactive
20:04:56.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.14.60:46841(127.19.14.60:46841)] cleaning up while in state READY due to: connection closed
20:04:56.219 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:56.219 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.14.61:35907
20:04:56.219 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:04:56.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:56.220106 19512 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 20506
20:04:56.254 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:56.254 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.19.14.61:35907(127.19.14.61:35907)] handling channelInactive
20:04:56.254 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.14.62:38303
20:04:56.254 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.14.61:35907(127.19.14.61:35907)] cleaning up while in state READY due to: connection closed
20:04:56.254 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:04:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:56.256610 19512 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:04:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data
20:04:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/logs
20:04:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data/info.pb
20:04:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.62:38303
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.62
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35405
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.62
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:04:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:04:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:56.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:56.633790 20674 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:56.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:56.634516 20674 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:56.635427 20674 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:56.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:56.670598 20674 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:04:56.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:56.670894 20674 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:56.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:56.671316 20674 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:56.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:56.671725 20674 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:56.672096 20674 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:56.672412 20674 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:56.672631 20674 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43011/jwks.json
20:04:56.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:56.715049 20674 master_runner.cc:387] Master server non-default flags:
20:04:56.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:56.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:56.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:56.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.62
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.62:38303
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data/info.pb
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.62
20:04:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35405
20:04:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/logs
20:04:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:56.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:04:56.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:56.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:04:56.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:04:56.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:56.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:56.716655 20674 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:56.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:56.718129 20674 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:56.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:56.735720 20682 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:58.138088 20681 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 20674
20:04:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:56.736255 20683 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:58.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:58.527822 20681 kernel_stack_watchdog.cc:198] Thread 20674 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
20:04:58.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:04:58.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:04:58.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:04:58.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:04:58.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:58.530387 20674 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.795s user 0.528s sys 1.099s
20:04:58.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:58.530802 20674 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.796s user 0.528s sys 1.099s
20:04:58.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:58.533114 20685 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:58.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:58.536440 20684 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1793 milliseconds
20:04:58.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.536543 20674 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.539249 20674 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:58.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.542686 20674 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:58.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.544237 20674 hybrid_clock.cc:648] HybridClock initialized: now 1781813098544179 us; error 62 us; skew 500 ppm
20:04:58.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.545404 20674 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:58 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813098, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:58.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.554692 20674 init.cc:377] Logged in from keytab as kudu/127.19.14.62@KRBTEST.COM (short username kudu)
20:04:58.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.561758 20674 webserver.cc:533] Webserver started at http://127.19.14.62:35405/ using document root <none> and password file <none>
20:04:58.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.562940 20674 fs_manager.cc:362] Metadata directory not provided
20:04:58.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.563226 20674 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:58.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.573221 20674 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.001s
20:04:58.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.579478 20700 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:58.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.581259 20674 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
20:04:58.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.581746 20674 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:04:58.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1acde45546ea4e3196a6946f6fc302c7"
20:04:58.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:28 on dist-test-slave-k9gl"
20:04:58.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.584498 20674 fs_report.cc:389] FS layout report
20:04:58.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:58.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:04:58.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:04:58.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data/data
20:04:58.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:58.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:58.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:58.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:58.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:58.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:58.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:58.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:58.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:58.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:58.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:58.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.637827 20674 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.650619 20674 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.651242 20674 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:58.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.746476 20674 rpc_server.cc:307] RPC server started. Bound to: 127.19.14.62:38303
20:04:58.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.746764 20752 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.14.62:38303 every 8 connection(s)
20:04:58.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:58 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813098, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.748483 19870 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:58.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.749838 20674 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data/info.pb
20:04:58.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.758978 19512 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 20674
20:04:58.759 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:58.759 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.14.60:46841
20:04:58.760 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:04:58.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.761768 19512 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:58.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:04:58.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:04:58.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/logs
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data/info.pb
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.60:46841
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.60
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45819
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.60
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.764813 20753 sys_catalog.cc:263] Verifying existing consensus state
20:04:58.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.774525 20753 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Bootstrap starting.
20:04:58.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.838588 20753 log.cc:826] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Log is configured to *not* fsync() on all Append() calls
20:04:58.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.869280 20753 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:58.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.870687 20753 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Bootstrap complete.
20:04:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.902338 20753 raft_consensus.cc:359] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.905395 20753 raft_consensus.cc:740] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1acde45546ea4e3196a6946f6fc302c7, State: Initialized, Role: FOLLOWER
20:04:58.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.906469 20753 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:04:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.910136 20757 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:04:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.911103 20757 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: This master's current role is: FOLLOWER
20:04:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.912310 20753 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: configured and running, proceeding with master startup.
20:04:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.955932 20768 catalog_manager.cc:1294] Loaded cluster ID: 65300b4ca4c54eb8939d73cea6dbaf4d
20:04:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.956950 20768 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: loading cluster ID for follower catalog manager: success
20:04:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.965796 20768 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: acquiring CA information for follower catalog manager: success
20:04:58.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:04:58 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813098, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.62@KRBTEST.COM for kudu/127.19.14.62@KRBTEST.COM
20:04:58.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:58.974570 20768 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:04:59.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:59.005025 20674 master_runner.cc:187] Error getting master registration for 127.19.14.61:35907: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111),
20:04:59.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:59.035166 20674 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:04:59.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:59.156556 20289 heartbeater.cc:344] Connected to a master server at 127.19.14.62:38303
20:04:59.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:59.164539 20718 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "5eb4b8df01144c9284949cab2ce0950c" instance_seqno: 1781813084768865) as {username='kudu', principal='kudu/127.19.14.3@KRBTEST.COM'} at 127.19.14.3:48611; Asking this server to re-register.
20:04:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:59.167348 20289 heartbeater.cc:461] Registering TS with master...
20:04:59.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:59.172049 20718 ts_manager.cc:194] Registered new tserver with Master: 5eb4b8df01144c9284949cab2ce0950c (127.19.14.3:41975)
20:04:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:59.235133 20755 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:59.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:59.235947 20755 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:59.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:59.236706 20755 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:59.279860 20755 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:04:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:59.280207 20755 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:59.280797 20755 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:59.281191 20755 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:59.281708 20755 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:59.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:59.282148 20755 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:59.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:59.282430 20755 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43011/jwks.json
20:04:59.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:59.338058 20755 master_runner.cc:387] Master server non-default flags:
20:04:59.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:59.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:04:59.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.60
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.60:46841
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data/info.pb
20:04:59.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.60
20:04:59.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45819
20:04:59.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:59.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:59.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:59.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:59.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:59.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/logs
20:04:59.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:59.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:59.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:59.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:59.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:59.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:04:59.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:59.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:04:59.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:04:59.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:59.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:59.340260 20755 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:59.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:59.342036 20755 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:59.361572 20779 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:59.500874 20142 heartbeater.cc:344] Connected to a master server at 127.19.14.62:38303
20:04:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:59.505491 20718 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a2f68907433b4dce88718ce628e2a595" instance_seqno: 1781813082057972) as {username='kudu', principal='kudu/127.19.14.2@KRBTEST.COM'} at 127.19.14.2:60103; Asking this server to re-register.
20:04:59.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:59.507982 20142 heartbeater.cc:461] Registering TS with master...
20:04:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:59.513221 20718 ts_manager.cc:194] Registered new tserver with Master: a2f68907433b4dce88718ce628e2a595 (127.19.14.2:42427)
20:04:59.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:59.683082 19992 heartbeater.cc:344] Connected to a master server at 127.19.14.62:38303
20:04:59.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:59.688198 20718 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "6509385f6f204a549d529840cc27b524" instance_seqno: 1781813079281773) as {username='kudu', principal='kudu/127.19.14.1@KRBTEST.COM'} at 127.19.14.1:45955; Asking this server to re-register.
20:04:59.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:59.689991 19992 heartbeater.cc:461] Registering TS with master...
20:04:59.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:04:59.694088 20718 ts_manager.cc:194] Registered new tserver with Master: 6509385f6f204a549d529840cc27b524 (127.19.14.1:46821)
20:05:00.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:00.202188 20789 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:05:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:00.203202 20789 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:05:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:00.225755 20789 leader_election.cc:290] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907), 91fa1f4bb837465ca07067a55ddc3b95 (127.19.14.60:46841)
20:05:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:00.241947 20702 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:05:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:00.260433 20702 leader_election.cc:336] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907): Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:05:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:00.272907 20702 leader_election.cc:336] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 91fa1f4bb837465ca07067a55ddc3b95 (127.19.14.60:46841): Network error: Client connection negotiation failed: client connection to 127.19.14.60:46841: connect: Connection refused (error 111)
20:05:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:00.273648 20702 leader_election.cc:304] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1acde45546ea4e3196a6946f6fc302c7; no voters: 350b6ce39ebc451c8560faceb2efafa7, 91fa1f4bb837465ca07067a55ddc3b95
20:05:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:00.275094 20789 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
20:05:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:01.780269 20781 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2406 milliseconds
20:05:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:00.763722 20778 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 20755
20:05:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:01.781718 20782 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:04:59.367168 20780 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:01.774008 20755 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.408s user 0.759s sys 1.386s
20:05:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:01.782711 20755 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.417s user 0.760s sys 1.386s
20:05:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:01.783056 20755 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:05:01.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:01.790673 20755 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:05:01.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:01.796958 20755 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:05:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:01.799223 20755 hybrid_clock.cc:648] HybridClock initialized: now 1781813101799189 us; error 61 us; skew 500 ppm
20:05:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:01.800859 20755 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:05:01.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:01 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813101, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:05:01.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:01.811678 20755 init.cc:377] Logged in from keytab as kudu/127.19.14.60@KRBTEST.COM (short username kudu)
20:05:01.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:01 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813101, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:05:01.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:01.819097 20020 init.cc:295] Successfully reacquired a new kerberos TGT
20:05:01.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:01.821040 20755 webserver.cc:533] Webserver started at http://127.19.14.60:45819/ using document root <none> and password file <none>
20:05:01.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:01.822551 20755 fs_manager.cc:362] Metadata directory not provided
20:05:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:01.822893 20755 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:05:01.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:01.837726 20755 fs_manager.cc:714] Time spent opening directory manager: real 0.009s user 0.006s sys 0.000s
20:05:01.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:01.844355 20805 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:05:01.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:01.846556 20755 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.002s sys 0.003s
20:05:01.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:01.846978 20755 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:05:01.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "91fa1f4bb837465ca07067a55ddc3b95"
20:05:01.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:33 on dist-test-slave-k9gl"
20:05:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:01.849476 20755 fs_report.cc:389] FS layout report
20:05:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:05:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:05:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:05:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data/data
20:05:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:05:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:05:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:05:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:05:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:05:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:05:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:05:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:05:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:05:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:05:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:05:01.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:01.944264 20755 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:05:01.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:01.951483 20755 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:05:01.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:01.951987 20755 kserver.cc:163] Server-wide thread pool size limit: 3276
20:05:02.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.052644 20852 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:05:02.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.053536 20852 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:05:02.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.058497 20852 leader_election.cc:290] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907), 91fa1f4bb837465ca07067a55ddc3b95 (127.19.14.60:46841)
20:05:02.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.069737 20755 rpc_server.cc:307] RPC server started. Bound to: 127.19.14.60:46841
20:05:02.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.069864 20860 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.14.60:46841 every 8 connection(s)
20:05:02.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:02.072450 20702 leader_election.cc:336] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907): Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.074739 20755 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data/info.pb
20:05:02.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.081773 19512 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 20755
20:05:02.082 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:05:02.082 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.14.61:35907
20:05:02.083 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.085351 19512 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/logs
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data/info.pb
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.61:35907
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.61
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37259
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.61
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:05:02.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.111654 20862 sys_catalog.cc:263] Verifying existing consensus state
20:05:02.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:02 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813098, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.62@KRBTEST.COM for kudu/127.19.14.60@KRBTEST.COM
20:05:02.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.130844 20862 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: Bootstrap starting.
20:05:02.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.193024 20862 log.cc:826] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: Log is configured to *not* fsync() on all Append() calls
20:05:02.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.232369 20862 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:05:02.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.233476 20862 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: Bootstrap complete.
20:05:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.276189 20862 raft_consensus.cc:359] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:05:02.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.280582 20862 raft_consensus.cc:740] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 91fa1f4bb837465ca07067a55ddc3b95, State: Initialized, Role: FOLLOWER
20:05:02.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.281883 20862 consensus_queue.cc:260] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:05:02.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.300098 20862 sys_catalog.cc:565] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: configured and running, proceeding with master startup.
20:05:02.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.300262 20832 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1acde45546ea4e3196a6946f6fc302c7" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "91fa1f4bb837465ca07067a55ddc3b95" is_pre_election: true
20:05:02.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.301430 20832 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1acde45546ea4e3196a6946f6fc302c7 in term 2.
20:05:02.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.306851 20702 leader_election.cc:304] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1acde45546ea4e3196a6946f6fc302c7, 91fa1f4bb837465ca07067a55ddc3b95; no voters: 350b6ce39ebc451c8560faceb2efafa7
20:05:02.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.308573 20852 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 2 FOLLOWER]: Leader pre-election won for term 3
20:05:02.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.308992 20852 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:05:02.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.309371 20852 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 2 FOLLOWER]: Advancing to term 3
20:05:02.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.322006 20852 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:05:02.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:02 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813094, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.3@KRBTEST.COM for kudu/127.19.14.60@KRBTEST.COM
20:05:02.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.298578 20867 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:05:02.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.335500 20867 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: This master's current role is: FOLLOWER
20:05:02.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.337915 20852 leader_election.cc:290] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 3 election: Requested vote from peers 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907), 91fa1f4bb837465ca07067a55ddc3b95 (127.19.14.60:46841)
20:05:02.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.340247 20832 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1acde45546ea4e3196a6946f6fc302c7" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "91fa1f4bb837465ca07067a55ddc3b95"
20:05:02.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:02.355302 20702 leader_election.cc:336] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907): Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:05:02.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.356494 20832 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 2 FOLLOWER]: Advancing to term 3
20:05:02.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.423134 20288 heartbeater.cc:344] Connected to a master server at 127.19.14.60:46841
20:05:02.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.419499 20832 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1acde45546ea4e3196a6946f6fc302c7 in term 3.
20:05:02.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.437016 20702 leader_election.cc:304] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1acde45546ea4e3196a6946f6fc302c7, 91fa1f4bb837465ca07067a55ddc3b95; no voters: 350b6ce39ebc451c8560faceb2efafa7
20:05:02.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.438246 20852 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 3 FOLLOWER]: Leader election won for term 3
20:05:02.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.448959 20852 raft_consensus.cc:697] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 3 LEADER]: Becoming Leader. State: Replica: 1acde45546ea4e3196a6946f6fc302c7, State: Running, Role: LEADER
20:05:02.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.450965 20852 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:05:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.478006 20881 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1acde45546ea4e3196a6946f6fc302c7. Latest consensus state: current_term: 3 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:05:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.479480 20881 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: This master's current role is: LEADER
20:05:02.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.480561 20880 catalog_manager.cc:1294] Loaded cluster ID: 65300b4ca4c54eb8939d73cea6dbaf4d
20:05:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.480904 20880 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: loading cluster ID for follower catalog manager: success
20:05:02.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.490725 20886 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
20:05:02.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.501753 20886 catalog_manager.cc:1519] Initializing Kudu cluster ID...
20:05:02.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.509334 20880 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: acquiring CA information for follower catalog manager: success
20:05:02.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.515594 20880 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:05:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:02 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813101, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.60@KRBTEST.COM for kudu/127.19.14.62@KRBTEST.COM
20:05:02.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.520347 20886 catalog_manager.cc:1294] Loaded cluster ID: 65300b4ca4c54eb8939d73cea6dbaf4d
20:05:02.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.520898 20886 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
20:05:02.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.530555 20886 catalog_manager.cc:1539] Loading token signing keys...
20:05:02.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.535904 20886 catalog_manager.cc:6080] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Loaded TSK: 0
20:05:02.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.537746 20886 catalog_manager.cc:1549] Initializing in-progress tserver states...
20:05:02.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.561676 20755 master_runner.cc:187] Error getting master registration for 127.19.14.61:35907: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111),
20:05:02.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.593010 20755 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:05:02.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.602137 20141 heartbeater.cc:344] Connected to a master server at 127.19.14.60:46841
20:05:02.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.611094 20822 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a2f68907433b4dce88718ce628e2a595" instance_seqno: 1781813082057972) as {username='kudu', principal='kudu/127.19.14.2@KRBTEST.COM'} at 127.19.14.2:53335; Asking this server to re-register.
20:05:02.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.613471 20141 heartbeater.cc:461] Registering TS with master...
20:05:02.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.617146 20822 ts_manager.cc:194] Registered new tserver with Master: a2f68907433b4dce88718ce628e2a595 (127.19.14.2:42427)
20:05:02.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:02.694669 20864 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:05:02.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:02.695480 20864 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:05:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:02.696524 20864 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:05:02.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:02.740460 20864 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:05:02.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:02.740847 20864 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:05:02.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:02.741483 20864 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:05:02.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:02.741855 20864 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:05:02.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:02.742395 20864 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:05:02.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:02.742745 20864 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:05:02.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:02.743130 20864 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43011/jwks.json
20:05:02.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.790333 19989 heartbeater.cc:344] Connected to a master server at 127.19.14.60:46841
20:05:02.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.797343 20822 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "6509385f6f204a549d529840cc27b524" instance_seqno: 1781813079281773) as {username='kudu', principal='kudu/127.19.14.1@KRBTEST.COM'} at 127.19.14.1:54003; Asking this server to re-register.
20:05:02.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.799744 19989 heartbeater.cc:461] Registering TS with master...
20:05:02.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.803341 20822 ts_manager.cc:194] Registered new tserver with Master: 6509385f6f204a549d529840cc27b524 (127.19.14.1:46821)
20:05:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.822286 20864 master_runner.cc:387] Master server non-default flags:
20:05:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:05:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data
20:05:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:05:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:05:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:05:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:05:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:05:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.61
20:05:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:05:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.61:35907
20:05:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:05:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data/info.pb
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.61
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37259
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/logs
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:05:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:05:02.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.824175 20864 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:05:02.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.825585 20864 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:05:02.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:02.847364 20897 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:02.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.994031 20832 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 3 FOLLOWER]: Refusing update from remote peer 1acde45546ea4e3196a6946f6fc302c7: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
20:05:02.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:02.996243 20881 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.001s
20:05:03.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:03.045395 20702 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 -> Peer 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907): Couldn't send request to peer 350b6ce39ebc451c8560faceb2efafa7. Status: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:05:03.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:03.060551 20903 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1acde45546ea4e3196a6946f6fc302c7. Latest consensus state: current_term: 3 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:05:03.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:03.062878 20903 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: This master's current role is: FOLLOWER
20:05:03.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:03.076908 20903 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:05:03.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:03.079361 20903 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: This master's current role is: FOLLOWER
20:05:03.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:03.082499 20881 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:05:03.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:03.083647 20881 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: This master's current role is: LEADER
20:05:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:03.432217 20822 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "5eb4b8df01144c9284949cab2ce0950c" instance_seqno: 1781813084768865) as {username='kudu', principal='kudu/127.19.14.3@KRBTEST.COM'} at 127.19.14.3:37023; Asking this server to re-register.
20:05:03.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:03.434958 20288 heartbeater.cc:461] Registering TS with master...
20:05:03.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:03.445513 20822 ts_manager.cc:194] Registered new tserver with Master: 5eb4b8df01144c9284949cab2ce0950c (127.19.14.3:41975)
20:05:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:03.710074 19992 heartbeater.cc:499] Master 127.19.14.62:38303 was elected leader, sending a full tablet report...
20:05:04.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.046289 20674 master_runner.cc:187] Error getting master registration for 127.19.14.61:35907: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111),
20:05:04.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.047009 20674 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:05:04.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.198469 20289 heartbeater.cc:499] Master 127.19.14.62:38303 was elected leader, sending a full tablet report...
20:05:04.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:04 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813104, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:05:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.417966 20170 init.cc:295] Successfully reacquired a new kerberos TGT
20:05:04.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:02.854432 20898 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:04.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:04.456122 20900 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:04.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:04.463559 20899 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1613 milliseconds
20:05:04.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.463644 20864 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:05:04.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.465327 20864 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:05:04.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.468614 20864 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:05:04.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.470165 20864 hybrid_clock.cc:648] HybridClock initialized: now 1781813104470112 us; error 80 us; skew 500 ppm
20:05:04.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.471349 20864 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:05:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:04 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813104, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:05:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.481019 20864 init.cc:377] Logged in from keytab as kudu/127.19.14.61@KRBTEST.COM (short username kudu)
20:05:04.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.488955 20864 webserver.cc:533] Webserver started at http://127.19.14.61:37259/ using document root <none> and password file <none>
20:05:04.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.490376 20864 fs_manager.cc:362] Metadata directory not provided
20:05:04.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.490718 20864 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:05:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.501580 20864 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.001s
20:05:04.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.508447 20926 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:05:04.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.510463 20864 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.006s sys 0.001s
20:05:04.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.511030 20864 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:05:04.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "350b6ce39ebc451c8560faceb2efafa7"
20:05:04.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:31 on dist-test-slave-k9gl"
20:05:04.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.514542 20864 fs_report.cc:389] FS layout report
20:05:04.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:05:04.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:05:04.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:05:04.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data/data
20:05:04.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:05:04.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:05:04.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:05:04.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:05:04.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:05:04.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:05:04.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:05:04.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:05:04.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:05:04.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:05:04.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:05:04.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.546211 20142 heartbeater.cc:499] Master 127.19.14.62:38303 was elected leader, sending a full tablet report...
20:05:04.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.586776 20864 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:05:04.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.599197 20864 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:05:04.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.599848 20864 kserver.cc:163] Server-wide thread pool size limit: 3276
20:05:04.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.762090 20864 rpc_server.cc:307] RPC server started. Bound to: 127.19.14.61:35907
20:05:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.762328 20980 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.14.61:35907 every 8 connection(s)
20:05:04.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.767735 20864 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data/info.pb
20:05:04.772 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:05:04.774 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.14.62:38303(127.19.14.62:38303)] connecting to peer
20:05:04.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.770988 19512 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 20864
20:05:04.791 [DEBUG - kudu-nio-3] (Connection.java:757) [peer master-127.19.14.62:38303(127.19.14.62:38303)] Successfully connected to peer
20:05:04.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.812192 20981 sys_catalog.cc:263] Verifying existing consensus state
20:05:04.818 [DEBUG - kudu-nio-3] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.19.14.62
20:05:04.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.846662 20981 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: Bootstrap starting.
20:05:04.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:04 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1781813096, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.14.62@KRBTEST.COM
20:05:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:04 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813098, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.1@KRBTEST.COM for kudu/127.19.14.61@KRBTEST.COM
20:05:04.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.944366 19988 heartbeater.cc:344] Connected to a master server at 127.19.14.61:35907
20:05:04.972 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x74467644, L:/127.0.0.1:45700 - R:/127.19.14.62:38303] using SASL/GSSAPI
20:05:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:04.975706 20981 log.cc:826] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: Log is configured to *not* fsync() on all Append() calls
20:05:05.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.001128 20981 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:05:05.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.002429 20981 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: Bootstrap complete.
20:05:05.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.026962 20981 raft_consensus.cc:359] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:05:05.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.029904 20981 raft_consensus.cc:740] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 350b6ce39ebc451c8560faceb2efafa7, State: Initialized, Role: FOLLOWER
20:05:05.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.031124 20981 consensus_queue.cc:260] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:05:05.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.034464 20989 sys_catalog.cc:455] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:05:05.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.035843 20989 sys_catalog.cc:458] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: This master's current role is: FOLLOWER
20:05:05.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.036698 20981 sys_catalog.cc:565] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: configured and running, proceeding with master startup.
20:05:05.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.086689 21000 catalog_manager.cc:1294] Loaded cluster ID: 65300b4ca4c54eb8939d73cea6dbaf4d
20:05:05.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.087066 21000 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: loading cluster ID for follower catalog manager: success
20:05:05.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.095741 21000 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: acquiring CA information for follower catalog manager: success
20:05:05.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.102346 21000 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:05:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:05 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813104, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.61@KRBTEST.COM for kudu/127.19.14.62@KRBTEST.COM
20:05:05.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.178834 20956 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [term 2 FOLLOWER]: Advancing to term 3
20:05:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.226147 21007 sys_catalog.cc:455] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:05:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.225309 20989 sys_catalog.cc:455] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1acde45546ea4e3196a6946f6fc302c7. Latest consensus state: current_term: 3 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:05:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.228269 21007 sys_catalog.cc:458] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: This master's current role is: FOLLOWER
20:05:05.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.228663 20989 sys_catalog.cc:458] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: This master's current role is: FOLLOWER
20:05:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.454913 20287 heartbeater.cc:344] Connected to a master server at 127.19.14.61:35907
20:05:05.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.463207 20946 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "5eb4b8df01144c9284949cab2ce0950c" instance_seqno: 1781813084768865) as {username='kudu', principal='kudu/127.19.14.3@KRBTEST.COM'} at 127.19.14.3:50765; Asking this server to re-register.
20:05:05.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.464963 20287 heartbeater.cc:461] Registering TS with master...
20:05:05.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.469955 20946 ts_manager.cc:194] Registered new tserver with Master: 5eb4b8df01144c9284949cab2ce0950c (127.19.14.3:41975)
20:05:05.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.681023 21011 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:05:05.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.686820 21011 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: This master's current role is: LEADER
20:05:05.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.695906 20140 heartbeater.cc:344] Connected to a master server at 127.19.14.61:35907
20:05:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.699491 20946 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a2f68907433b4dce88718ce628e2a595" instance_seqno: 1781813082057972) as {username='kudu', principal='kudu/127.19.14.2@KRBTEST.COM'} at 127.19.14.2:54041; Asking this server to re-register.
20:05:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.701591 20140 heartbeater.cc:461] Registering TS with master...
20:05:05.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.706081 20946 ts_manager.cc:194] Registered new tserver with Master: a2f68907433b4dce88718ce628e2a595 (127.19.14.2:42427)
20:05:05.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.954288 20946 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "6509385f6f204a549d529840cc27b524" instance_seqno: 1781813079281773) as {username='kudu', principal='kudu/127.19.14.1@KRBTEST.COM'} at 127.19.14.1:39183; Asking this server to re-register.
20:05:05.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.956792 19988 heartbeater.cc:461] Registering TS with master...
20:05:05.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:05.961203 20946 ts_manager.cc:194] Registered new tserver with Master: 6509385f6f204a549d529840cc27b524 (127.19.14.1:46821)
20:05:05.983 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:05:05.983 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:05:06.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:06 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813106, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:05:06.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:05:06.022 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:05:06.022 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc
20:05:06.024 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:05:06.025 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.14.62:38303
20:05:06.025 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:05:06.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:06.025977 19512 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 20674
20:05:06.064 [DEBUG - kudu-nio-3] (Connection.java:239) [peer master-127.19.14.62:38303(127.19.14.62:38303)] handling channelInactive
20:05:06.064 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.14.62:38303(127.19.14.62:38303)] cleaning up while in state READY due to: connection closed
20:05:06.069 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:05:06.069 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.14.60:46841
20:05:06.070 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:05:06.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:06.070436 19512 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 20755
20:05:06.111 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:05:06.111 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.14.61:35907
20:05:06.111 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:05:06.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:06.114317 19512 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 20864
20:05:06.149 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:05:06.149 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.14.62:38303
20:05:06.150 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:05:06.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:06.151742 19512 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:05:06.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:05:06.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:05:06.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data
20:05:06.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:05:06.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:05:06.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:05:06.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:06.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:06.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:06.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/logs
20:05:06.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data/info.pb
20:05:06.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:06.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:05:06.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:05:06.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:05:06.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:05:06.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:06.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:06.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:06.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:05:06.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:05:06.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:05:06.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.62:38303
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.62
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35405
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.62
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:05:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:05:06.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:06.525566 21017 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:05:06.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:06.526378 21017 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:05:06.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:06.527271 21017 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:05:06.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:06.568135 21017 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:05:06.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:06.568509 21017 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:05:06.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:06.569010 21017 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:05:06.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:06.569383 21017 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:05:06.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:06.569737 21017 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:05:06.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:06.570091 21017 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:05:06.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:06.570358 21017 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43011/jwks.json
20:05:06.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:06.614869 21017 master_runner.cc:387] Master server non-default flags:
20:05:06.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:06.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:05:06.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:06.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:06.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data
20:05:06.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:05:06.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:05:06.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:05:06.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.62
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.62:38303
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data/info.pb
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.62
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35405
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/logs
20:05:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:05:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:05:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:05:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:05:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:05:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:05:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:05:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:05:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:05:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:06.616879 21017 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:05:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:06.618461 21017 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:05:06.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:06.635675 21025 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:08.042389 21024 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 21017
20:05:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:06.636780 21026 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:08.199522 21017 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.565s user 0.592s sys 0.959s
20:05:08.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:08.199996 21017 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.566s user 0.592s sys 0.959s
20:05:08.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:08.209299 21036 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:08.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.209446 21017 server_base.cc:1061] running on GCE node
20:05:08.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.211078 21017 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:05:08.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.214918 21017 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:05:08.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.216436 21017 hybrid_clock.cc:648] HybridClock initialized: now 1781813108216331 us; error 130 us; skew 500 ppm
20:05:08.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.217653 21017 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:05:08.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:08 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813108, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:05:08.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.226505 21017 init.cc:377] Logged in from keytab as kudu/127.19.14.62@KRBTEST.COM (short username kudu)
20:05:08.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.234472 21017 webserver.cc:533] Webserver started at http://127.19.14.62:35405/ using document root <none> and password file <none>
20:05:08.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.235666 21017 fs_manager.cc:362] Metadata directory not provided
20:05:08.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.235924 21017 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:05:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.250223 21017 fs_manager.cc:714] Time spent opening directory manager: real 0.010s user 0.009s sys 0.002s
20:05:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.258550 21043 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:05:08.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.260465 21017 fs_manager.cc:730] Time spent opening block manager: real 0.007s user 0.001s sys 0.004s
20:05:08.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.261016 21017 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:05:08.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1acde45546ea4e3196a6946f6fc302c7"
20:05:08.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:28 on dist-test-slave-k9gl"
20:05:08.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.265069 21017 fs_report.cc:389] FS layout report
20:05:08.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:05:08.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:05:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/wal
20:05:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data/data
20:05:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:05:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:05:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:05:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:05:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:05:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:05:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:05:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:05:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:05:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:05:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:05:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.338451 21017 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:05:08.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.344556 21017 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:05:08.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.345214 21017 kserver.cc:163] Server-wide thread pool size limit: 3276
20:05:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.436213 21017 rpc_server.cc:307] RPC server started. Bound to: 127.19.14.62:38303
20:05:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.436280 21094 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.14.62:38303 every 8 connection(s)
20:05:08.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.439291 21017 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-0/data/info.pb
20:05:08.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.446051 19512 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 21017
20:05:08.448 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:05:08.449 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.14.60:46841
20:05:08.449 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:05:08.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.451634 19512 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:05:08.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:05:08.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:05:08.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data
20:05:08.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:05:08.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:05:08.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:05:08.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:08.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:08.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/logs
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data/info.pb
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.60:46841
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.60
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45819
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.60
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:05:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:05:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:05:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:05:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:05:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:05:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:05:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:05:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.454625 21095 sys_catalog.cc:263] Verifying existing consensus state
20:05:08.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.465090 21095 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Bootstrap starting.
20:05:08.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.537707 21095 log.cc:826] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Log is configured to *not* fsync() on all Append() calls
20:05:08.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.582056 21095 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:05:08.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.583801 21095 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Bootstrap complete.
20:05:08.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:08 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813101, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.2@KRBTEST.COM for kudu/127.19.14.62@KRBTEST.COM
20:05:08.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.630648 21095 raft_consensus.cc:359] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:05:08.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.637317 21095 raft_consensus.cc:740] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1acde45546ea4e3196a6946f6fc302c7, State: Initialized, Role: FOLLOWER
20:05:08.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.638969 21095 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:05:08.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.645535 21101 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:05:08.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.648393 21101 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: This master's current role is: FOLLOWER
20:05:08.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.650804 21095 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: configured and running, proceeding with master startup.
20:05:08.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.667528 20142 heartbeater.cc:344] Connected to a master server at 127.19.14.62:38303
20:05:08.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.710711 21112 catalog_manager.cc:1294] Loaded cluster ID: 65300b4ca4c54eb8939d73cea6dbaf4d
20:05:08.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.711621 21112 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: loading cluster ID for follower catalog manager: success
20:05:08.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.721861 21112 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: acquiring CA information for follower catalog manager: success
20:05:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.729687 21112 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:05:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:08 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813108, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.62@KRBTEST.COM for kudu/127.19.14.62@KRBTEST.COM
20:05:08.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:08 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813108, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:05:08.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.756699 19870 init.cc:295] Successfully reacquired a new kerberos TGT
20:05:08.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.792549 19992 heartbeater.cc:344] Connected to a master server at 127.19.14.62:38303
20:05:08.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.760362 21017 master_runner.cc:187] Error getting master registration for 127.19.14.61:35907: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111),
20:05:08.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.810325 21017 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:05:08.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.812563 21060 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "6509385f6f204a549d529840cc27b524" instance_seqno: 1781813079281773) as {username='kudu', principal='kudu/127.19.14.1@KRBTEST.COM'} at 127.19.14.1:52471; Asking this server to re-register.
20:05:08.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.814738 19992 heartbeater.cc:461] Registering TS with master...
20:05:08.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:08.819828 21060 ts_manager.cc:194] Registered new tserver with Master: 6509385f6f204a549d529840cc27b524 (127.19.14.1:46821)
20:05:09.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:09.020722 21097 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:05:09.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:09.021731 21097 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:05:09.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:09.022632 21097 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:05:09.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:09.066923 21097 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:05:09.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:09.067327 21097 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:05:09.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:09.067827 21097 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:05:09.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:09.068221 21097 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:05:09.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:09.068581 21097 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:05:09.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:09.068960 21097 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:05:09.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:09.069257 21097 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43011/jwks.json
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:09.134572 21097 master_runner.cc:387] Master server non-default flags:
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.60
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.60:46841
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:05:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data/info.pb
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.60
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45819
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/logs
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:05:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:05:09.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:09.136768 21097 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:05:09.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:09.138253 21097 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:05:09.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:09.155830 21122 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:09.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:09.287421 20289 heartbeater.cc:344] Connected to a master server at 127.19.14.62:38303
20:05:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:09.292879 21060 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "5eb4b8df01144c9284949cab2ce0950c" instance_seqno: 1781813084768865) as {username='kudu', principal='kudu/127.19.14.3@KRBTEST.COM'} at 127.19.14.3:50583; Asking this server to re-register.
20:05:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:09.295908 20289 heartbeater.cc:461] Registering TS with master...
20:05:09.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:09.302037 21060 ts_manager.cc:194] Registered new tserver with Master: 5eb4b8df01144c9284949cab2ce0950c (127.19.14.3:41975)
20:05:09.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:09.678341 21060 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a2f68907433b4dce88718ce628e2a595" instance_seqno: 1781813082057972) as {username='kudu', principal='kudu/127.19.14.2@KRBTEST.COM'} at 127.19.14.2:53649; Asking this server to re-register.
20:05:09.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:09.681672 20142 heartbeater.cc:461] Registering TS with master...
20:05:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:09.688659 21059 ts_manager.cc:194] Registered new tserver with Master: a2f68907433b4dce88718ce628e2a595 (127.19.14.2:42427)
20:05:10.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:10.229811 21133 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:05:10.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:10.233582 21133 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:05:10.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:10.265659 21045 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:05:10.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:10.288843 21133 leader_election.cc:290] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907), 91fa1f4bb837465ca07067a55ddc3b95 (127.19.14.60:46841)
20:05:10.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:10.303900 21045 leader_election.cc:336] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907): Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:05:10.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:10.326960 21045 leader_election.cc:336] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 91fa1f4bb837465ca07067a55ddc3b95 (127.19.14.60:46841): Network error: Client connection negotiation failed: client connection to 127.19.14.60:46841: connect: Connection refused (error 111)
20:05:10.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:10.329926 21045 leader_election.cc:304] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1acde45546ea4e3196a6946f6fc302c7; no voters: 350b6ce39ebc451c8560faceb2efafa7, 91fa1f4bb837465ca07067a55ddc3b95
20:05:10.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:10.334764 21133 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
20:05:11.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:11.162384 18938 debug-util.cc:398] Leaking SignalData structure 0x7b080008ab40 after lost signal to thread 18873
20:05:11.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:11.163631 18938 debug-util.cc:398] Leaking SignalData structure 0x7b08000982a0 after lost signal to thread 18941
20:05:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:10.559432 21121 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 21097
20:05:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:11.216470 21121 kernel_stack_watchdog.cc:198] Thread 21097 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 397ms:
20:05:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:05:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:05:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:05:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:05:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:09.157665 21123 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:11.218448 21097 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.061s user 0.002s sys 0.000s
20:05:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:11.219002 21097 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.061s user 0.002s sys 0.000s
20:05:11.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:11.222504 21125 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.224706 21097 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:05:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:11.225407 21124 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2064 milliseconds
20:05:11.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.227279 21097 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:05:11.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.231938 21097 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:05:11.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.233551 21097 hybrid_clock.cc:648] HybridClock initialized: now 1781813111233446 us; error 90 us; skew 500 ppm
20:05:11.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.235337 21097 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:05:11.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:11 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813111, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:05:11.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.247637 21097 init.cc:377] Logged in from keytab as kudu/127.19.14.60@KRBTEST.COM (short username kudu)
20:05:11.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.255358 21097 webserver.cc:533] Webserver started at http://127.19.14.60:45819/ using document root <none> and password file <none>
20:05:11.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.257202 21097 fs_manager.cc:362] Metadata directory not provided
20:05:11.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.257659 21097 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:05:11.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.282524 21097 fs_manager.cc:714] Time spent opening directory manager: real 0.020s user 0.006s sys 0.008s
20:05:11.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.299886 21148 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:05:11.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.301994 21097 fs_manager.cc:730] Time spent opening block manager: real 0.016s user 0.010s sys 0.004s
20:05:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.302672 21097 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:05:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "91fa1f4bb837465ca07067a55ddc3b95"
20:05:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:33 on dist-test-slave-k9gl"
20:05:11.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.305914 21097 fs_report.cc:389] FS layout report
20:05:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:05:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:05:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/wal
20:05:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data/data
20:05:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:05:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:05:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:05:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:05:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:05:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:05:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:05:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:05:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:05:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:05:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:05:11.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.399847 21097 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:05:11.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.406023 21097 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:05:11.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.406572 21097 kserver.cc:163] Server-wide thread pool size limit: 3276
20:05:11.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.573558 21097 rpc_server.cc:307] RPC server started. Bound to: 127.19.14.60:46841
20:05:11.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.574143 21201 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.14.60:46841 every 8 connection(s)
20:05:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.577780 21097 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-2/data/info.pb
20:05:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.584096 19512 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 21097
20:05:11.585 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:05:11.585 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.14.61:35907
20:05:11.585 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:05:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.587822 19512 external_mini_cluster.cc:1372] Running /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:05:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu
20:05:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:05:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data
20:05:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:05:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:05:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:05:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/logs
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data/info.pb
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.61:35907
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.61
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37259
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.61
20:05:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:05:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:05:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:05:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:05:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:05:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:05:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:05:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:05:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:05:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:05:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:05:11.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.603281 21202 sys_catalog.cc:263] Verifying existing consensus state
20:05:11.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.612339 21202 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: Bootstrap starting.
20:05:11.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.672693 21202 log.cc:826] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: Log is configured to *not* fsync() on all Append() calls
20:05:11.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.722115 21202 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:05:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.723168 21202 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: Bootstrap complete.
20:05:11.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.769511 21202 raft_consensus.cc:359] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:05:11.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.773121 21202 raft_consensus.cc:740] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 91fa1f4bb837465ca07067a55ddc3b95, State: Initialized, Role: FOLLOWER
20:05:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.774170 21202 consensus_queue.cc:260] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:05:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.792161 21208 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:05:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.793174 21208 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: This master's current role is: FOLLOWER
20:05:11.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.794950 21202 sys_catalog.cc:565] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: configured and running, proceeding with master startup.
20:05:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:11.804374 21207 negotiation.cc:336] Failed RPC negotiation. Trace:
20:05:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:05:11.757498 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.19.14.2:56121 (local address 127.19.14.60:46841)
20:05:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:05:11.772980 (+ 15482us) server_negotiation.cc:207] Beginning negotiation
20:05:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:05:11.783370 (+ 10390us) server_negotiation.cc:400] Waiting for connection header
20:05:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:05:11.783517 (+ 147us) server_negotiation.cc:408] Connection header received
20:05:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:05:11.784420 (+ 903us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:05:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:05:11.784479 (+ 59us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:05:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:05:11.785354 (+ 875us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:05:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:05:11.785997 (+ 643us) server_negotiation.cc:227] Negotiated authn=SASL
20:05:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:05:11.798039 (+ 12042us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.19.14.2:56121: BlockingRecv error: recv got EOF from 127.19.14.2:56121 (error 108)
20:05:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":14461,"thread_start_us":15946,"threads_started":1}
20:05:11.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:11 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813111, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:05:11.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.824961 20020 init.cc:295] Successfully reacquired a new kerberos TGT
20:05:11.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:11.926774 21220 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:05:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:11.929204 19215 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8b60 after lost signal to thread 19087
20:05:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:11.930444 19215 debug-util.cc:398] Leaking SignalData structure 0x7b08000adec0 after lost signal to thread 19218
20:05:11.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:11 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813111, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.60@KRBTEST.COM for kudu/127.19.14.62@KRBTEST.COM
20:05:12.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.035709 21097 master_runner.cc:187] Error getting master registration for 127.19.14.61:35907: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111),
20:05:12.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.087649 21097 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:05:12.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:12 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813108, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.1@KRBTEST.COM for kudu/127.19.14.60@KRBTEST.COM
20:05:12.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.360314 19989 heartbeater.cc:344] Connected to a master server at 127.19.14.60:46841
20:05:12.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.371268 21166 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "6509385f6f204a549d529840cc27b524" instance_seqno: 1781813079281773) as {username='kudu', principal='kudu/127.19.14.1@KRBTEST.COM'} at 127.19.14.1:57743; Asking this server to re-register.
20:05:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.374778 19989 heartbeater.cc:461] Registering TS with master...
20:05:12.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.380714 21165 ts_manager.cc:194] Registered new tserver with Master: 6509385f6f204a549d529840cc27b524 (127.19.14.1:46821)
20:05:12.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.391881 21227 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:05:12.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.392784 21227 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:05:12.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.401645 21227 leader_election.cc:290] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907), 91fa1f4bb837465ca07067a55ddc3b95 (127.19.14.60:46841)
20:05:12.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:12.442409 21045 leader_election.cc:336] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907): Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:05:12.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:12 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813108, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.62@KRBTEST.COM for kudu/127.19.14.60@KRBTEST.COM
20:05:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.464380 21177 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1acde45546ea4e3196a6946f6fc302c7" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "91fa1f4bb837465ca07067a55ddc3b95" is_pre_election: true
20:05:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.465472 21177 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1acde45546ea4e3196a6946f6fc302c7 in term 3.
20:05:12.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.467072 21045 leader_election.cc:304] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1acde45546ea4e3196a6946f6fc302c7, 91fa1f4bb837465ca07067a55ddc3b95; no voters: 350b6ce39ebc451c8560faceb2efafa7
20:05:12.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.468218 21227 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 3 FOLLOWER]: Leader pre-election won for term 4
20:05:12.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.468569 21227 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:05:12.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.468959 21227 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 3 FOLLOWER]: Advancing to term 4
20:05:12.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.485792 21227 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:05:12.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:12.492792 21045 leader_election.cc:336] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907): Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111)
20:05:12.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.495116 21177 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1acde45546ea4e3196a6946f6fc302c7" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "91fa1f4bb837465ca07067a55ddc3b95"
20:05:12.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.495740 21177 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 3 FOLLOWER]: Advancing to term 4
20:05:12.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.507395 21177 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1acde45546ea4e3196a6946f6fc302c7 in term 4.
20:05:12.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.509750 21227 leader_election.cc:290] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 4 election: Requested vote from peers 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907), 91fa1f4bb837465ca07067a55ddc3b95 (127.19.14.60:46841)
20:05:12.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.526110 21045 leader_election.cc:304] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1acde45546ea4e3196a6946f6fc302c7, 91fa1f4bb837465ca07067a55ddc3b95; no voters: 350b6ce39ebc451c8560faceb2efafa7
20:05:12.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.527813 21227 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 4 FOLLOWER]: Leader election won for term 4
20:05:12.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.542737 21227 raft_consensus.cc:697] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [term 4 LEADER]: Becoming Leader. State: Replica: 1acde45546ea4e3196a6946f6fc302c7, State: Running, Role: LEADER
20:05:12.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.545229 21227 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:05:12.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:12 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813104, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.3@KRBTEST.COM for kudu/127.19.14.60@KRBTEST.COM
20:05:12.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.601718 21236 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1acde45546ea4e3196a6946f6fc302c7. Latest consensus state: current_term: 4 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:05:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.603255 21236 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: This master's current role is: LEADER
20:05:12.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.606599 21239 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
20:05:12.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.625173 20288 heartbeater.cc:344] Connected to a master server at 127.19.14.60:46841
20:05:12.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.628860 21166 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "5eb4b8df01144c9284949cab2ce0950c" instance_seqno: 1781813084768865) as {username='kudu', principal='kudu/127.19.14.3@KRBTEST.COM'} at 127.19.14.3:60337; Asking this server to re-register.
20:05:12.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.630625 20288 heartbeater.cc:461] Registering TS with master...
20:05:12.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.634270 21166 ts_manager.cc:194] Registered new tserver with Master: 5eb4b8df01144c9284949cab2ce0950c (127.19.14.3:41975)
20:05:12.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.639494 21239 catalog_manager.cc:1519] Initializing Kudu cluster ID...
20:05:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.644588 21239 catalog_manager.cc:1294] Loaded cluster ID: 65300b4ca4c54eb8939d73cea6dbaf4d
20:05:12.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.656822 21239 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
20:05:12.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.661844 21239 catalog_manager.cc:1539] Loading token signing keys...
20:05:12.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.666594 21239 catalog_manager.cc:6080] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7: Loaded TSK: 0
20:05:12.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.677924 21239 catalog_manager.cc:1549] Initializing in-progress tserver states...
20:05:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:12.748487 21204 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:05:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:12.749599 21204 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:05:12.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:12.750931 21204 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:05:12.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:12 dist-test-slave-k9gl krb5kdc[19524](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813111, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.2@KRBTEST.COM for kudu/127.19.14.60@KRBTEST.COM
20:05:12.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.864039 20141 heartbeater.cc:344] Connected to a master server at 127.19.14.60:46841
20:05:12.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.868111 21166 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a2f68907433b4dce88718ce628e2a595" instance_seqno: 1781813082057972) as {username='kudu', principal='kudu/127.19.14.2@KRBTEST.COM'} at 127.19.14.2:55385; Asking this server to re-register.
20:05:12.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.869884 20141 heartbeater.cc:461] Registering TS with master...
20:05:12.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.873449 21166 ts_manager.cc:194] Registered new tserver with Master: a2f68907433b4dce88718ce628e2a595 (127.19.14.2:42427)
20:05:12.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:12.905543 21204 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:05:12.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:12.906405 21204 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:05:12.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:12.907125 21204 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:05:12.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:12.907675 21204 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:05:12.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:12.909704 21204 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:05:12.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:12.910246 21204 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:05:12.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:12.910712 21204 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43011/jwks.json
20:05:12.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.956573 21220 catalog_manager.cc:1294] Loaded cluster ID: 65300b4ca4c54eb8939d73cea6dbaf4d
20:05:12.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.956913 21220 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: loading cluster ID for follower catalog manager: success
20:05:12.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.979699 21220 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: acquiring CA information for follower catalog manager: success
20:05:12.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.998307 21177 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [term 4 FOLLOWER]: Refusing update from remote peer 1acde45546ea4e3196a6946f6fc302c7: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
20:05:12.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:12.989802 21220 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:05:13.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:13.000269 21236 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:13.003720 21204 master_runner.cc:387] Master server non-default flags:
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.14.20:42649
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.14.62:38303,127.19.14.61:35907,127.19.14.60:46841
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/krb5kdc/kudu.keytab
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.14.61
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.14.61:35907
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43011/jwks.json
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:05:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data/info.pb
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.14.61
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37259
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/logs
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 19:43:23 UTC on e7f111948823
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12459
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:05:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:13.006175 21204 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:05:13.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:13.007964 21204 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:05:13.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:13.046105 21251 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:13.069747 21227 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:05:13.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:13.070585 21227 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 [sys.catalog]: This master's current role is: LEADER
20:05:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:13.088176 21256 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:05:13.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:13.082631 21245 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1acde45546ea4e3196a6946f6fc302c7. Latest consensus state: current_term: 4 leader_uuid: "1acde45546ea4e3196a6946f6fc302c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:05:13.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:13.095265 21245 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: This master's current role is: FOLLOWER
20:05:13.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:13.094482 21256 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91fa1f4bb837465ca07067a55ddc3b95 [sys.catalog]: This master's current role is: FOLLOWER
20:05:13.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:13.100710 21045 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 -> Peer 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907): Couldn't send request to peer 350b6ce39ebc451c8560faceb2efafa7. Status: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:05:13.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:13.709681 20142 heartbeater.cc:499] Master 127.19.14.62:38303 was elected leader, sending a full tablet report...
20:05:13.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:13.818564 21017 master_runner.cc:187] Error getting master registration for 127.19.14.61:35907: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111),
20:05:13.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:13.819051 21017 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:05:13.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:13.868238 19992 heartbeater.cc:499] Master 127.19.14.62:38303 was elected leader, sending a full tablet report...
20:05:14.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:14.326049 20289 heartbeater.cc:499] Master 127.19.14.62:38303 was elected leader, sending a full tablet report...
20:05:14.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:14 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813114, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:05:14.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:14.423659 20170 init.cc:295] Successfully reacquired a new kerberos TGT
20:05:15.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:15.536938 21045 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111) [suppressed 8 similar messages]
20:05:15.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:15.540273 21045 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1acde45546ea4e3196a6946f6fc302c7 -> Peer 350b6ce39ebc451c8560faceb2efafa7 (127.19.14.61:35907): Couldn't send request to peer 350b6ce39ebc451c8560faceb2efafa7. Status: Network error: Client connection negotiation failed: client connection to 127.19.14.61:35907: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:05:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:14.432158 21246 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 21204
20:05:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:15.630589 21246 kernel_stack_watchdog.cc:198] Thread 21204 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 379ms:
20:05:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:05:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:05:15.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:05:15.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:05:15.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:13.049681 21248 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:15.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:15.637601 21204 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.587s user 0.003s sys 0.000s
20:05:15.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:15.638075 21204 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.587s user 0.003s sys 0.000s
20:05:15.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:15.638926 21252 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2581 milliseconds
20:05:15.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:15.649328 21254 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:15.649433 21204 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:05:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:15.651315 21204 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:05:15.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:15.674217 21204 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:05:15.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:15.678131 21204 hybrid_clock.cc:648] HybridClock initialized: now 1781813115678062 us; error 85 us; skew 500 ppm
20:05:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:15.679546 21204 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:05:15.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 20:05:15 dist-test-slave-k9gl krb5kdc[19524](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781813115, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.14.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:05:15.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:15.690294 21204 init.cc:377] Logged in from keytab as kudu/127.19.14.61@KRBTEST.COM (short username kudu)
20:05:15.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:15.734622 21204 webserver.cc:533] Webserver started at http://127.19.14.61:37259/ using document root <none> and password file <none>
20:05:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:15.736088 21204 fs_manager.cc:362] Metadata directory not provided
20:05:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:15.736402 21204 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:05:15.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:15.760828 21204 fs_manager.cc:714] Time spent opening directory manager: real 0.018s user 0.013s sys 0.004s
20:05:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:15.775862 21278 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:05:15.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:15.778194 21204 fs_manager.cc:730] Time spent opening block manager: real 0.013s user 0.009s sys 0.003s
20:05:15.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:15.778854 21204 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data,/tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:05:15.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "350b6ce39ebc451c8560faceb2efafa7"
20:05:15.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 20:04:31 on dist-test-slave-k9gl"
20:05:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:15.782454 21204 fs_report.cc:389] FS layout report
20:05:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:05:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:05:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/wal
20:05:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data/data
20:05:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:05:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:05:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:05:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:05:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:05:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:05:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:05:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:05:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:05:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:05:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:05:16.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.002440 21204 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:05:16.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.019091 21204 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:05:16.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.019790 21204 kserver.cc:163] Server-wide thread pool size limit: 3276
20:05:16.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.210959 21204 rpc_server.cc:307] RPC server started. Bound to: 127.19.14.61:35907
20:05:16.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.211016 21332 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.14.61:35907 every 8 connection(s)
20:05:16.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.215477 21204 server_base.cc:1193] Dumped server information to /tmp/dist-test-task7Ze3A5/test-tmp/mini-kudu-cluster9735972843491530151/master-1/data/info.pb
20:05:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.217674 19512 external_mini_cluster.cc:1434] Started /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu as pid 21204
20:05:16.219 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:05:16.220 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.14.62:38303(127.19.14.62:38303)] connecting to peer
20:05:16.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.239605 21333 sys_catalog.cc:263] Verifying existing consensus state
20:05:16.241 [DEBUG - kudu-nio-4] (Connection.java:757) [peer master-127.19.14.62:38303(127.19.14.62:38303)] Successfully connected to peer
20:05:16.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.254843 21333 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: Bootstrap starting.
20:05:16.279 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.14.62:38303(127.19.14.62:38303)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
20:05:16.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 20:05:16.282145 21336 negotiation.cc:336] Failed RPC negotiation. Trace:
20:05:16.283 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.19.14.61:35907(127.19.14.61:35907)] disconnecting while in state TERMINATED
20:05:16.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:05:16.240898 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:37888 (local address 127.19.14.62:38303)
20:05:16.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:05:16.254063 (+ 13165us) server_negotiation.cc:207] Beginning negotiation
20:05:16.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:05:16.254094 (+ 31us) server_negotiation.cc:400] Waiting for connection header
20:05:16.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:05:16.263464 (+ 9370us) server_negotiation.cc:408] Connection header received
20:05:16.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:05:16.274024 (+ 10560us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:05:16.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:05:16.274065 (+ 41us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:05:16.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:05:16.274359 (+ 294us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:05:16.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:05:16.274552 (+ 193us) server_negotiation.cc:227] Negotiated authn=SASL
20:05:16.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0618 20:05:16.280572 (+ 6020us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37888: BlockingRecv error: recv got EOF from 127.0.0.1:37888 (error 108)
20:05:16.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":12516,"thread_start_us":11769,"threads_started":1}
20:05:16.284 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.19.14.60:46841(127.19.14.60:46841)] disconnecting while in state TERMINATED
20:05:16.285 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.19.14.62:38303(127.19.14.62:38303)] disconnecting while in state TERMINATED
20:05:16.286 [DEBUG - kudu-nio-4] (Connection.java:239) [peer master-127.19.14.62:38303(127.19.14.62:38303)] handling channelInactive
20:05:16.287 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:05:16.312 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:05:16.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.331660 19512 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 19853
20:05:16.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.347631 21333 log.cc:826] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: Log is configured to *not* fsync() on all Append() calls
20:05:16.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.375977 19512 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 20002
20:05:16.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.391093 21333 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:05:16.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.392717 21333 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7: Bootstrap complete.
20:05:16.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.434835 19512 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 20155
20:05:16.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.437721 21333 raft_consensus.cc:359] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:05:16.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.441777 21333 raft_consensus.cc:740] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 350b6ce39ebc451c8560faceb2efafa7, State: Initialized, Role: FOLLOWER
20:05:16.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.443399 21333 consensus_queue.cc:260] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } }
20:05:16.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.483070 21345 sys_catalog.cc:455] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1acde45546ea4e3196a6946f6fc302c7" member_type: VOTER last_known_addr { host: "127.19.14.62" port: 38303 } } peers { permanent_uuid: "350b6ce39ebc451c8560faceb2efafa7" member_type: VOTER last_known_addr { host: "127.19.14.61" port: 35907 } } peers { permanent_uuid: "91fa1f4bb837465ca07067a55ddc3b95" member_type: VOTER last_known_addr { host: "127.19.14.60" port: 46841 } } }
20:05:16.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.484336 21345 sys_catalog.cc:458] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: This master's current role is: FOLLOWER
20:05:16.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.485602 21333 sys_catalog.cc:565] T 00000000000000000000000000000000 P 350b6ce39ebc451c8560faceb2efafa7 [sys.catalog]: configured and running, proceeding with master startup.
20:05:16.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.494006 19512 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 21017
20:05:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.576653 19512 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 21204
20:05:16.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:16.630307 19512 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 21097
20:05:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-18T20:05:17Z chronyd exiting
20:05:19.004 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:05:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:19.008371 18787 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 19086
20:05:19.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:19.048300 18787 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 19228
20:05:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:19.079257 18787 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 19369
20:05:19.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:19.114176 18787 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 18799
20:05:19.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:19.154779 18787 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 18869
20:05:19.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 20:05:19.197494 18787 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task7Ze3A5/build/tsan/bin/kudu with pid 18944
20:05:20.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-18T20:05:20Z chronyd exiting
20:05:21.490 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more