02:08:22.243 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:08:22.243 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskoZkKpr/test-tmp
02:08:22.243 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:08:22.244 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskoZkKpr/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:08:22.246 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:08:22.588 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:08:22.588 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:08:22.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:22.591579 11953 mini_oidc.cc:150] Starting JWKS server
02:08:22.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:22.594785 11953 webserver.cc:492] Webserver started at http://127.0.0.1:34447/ using document root <none> and password file <none>
02:08:22.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:22.595199 11953 mini_oidc.cc:186] Starting OIDC Discovery server
02:08:22.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:22.596875 11953 webserver.cc:492] Webserver started at http://127.0.0.1:42699/ using document root <none> and password file <none>
02:08:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-03T02:08:22Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:08:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-03T02:08:22Z Disabled control of system clock
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:22.654134 11953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-0/wal
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-0/data
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-0/logs
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-0/data/info.pb
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.172.126:44025
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.172.126
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.172.84:43157
02:08:22.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:22.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:22.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:22.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:22.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.172.126:44025,127.11.172.125:40317,127.11.172.124:33559
02:08:22.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:22.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34447/jwks.json with env {}
02:08:22.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:22.946163 11965 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:22.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:22.946735 11965 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:22.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:22.947191 11965 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:22.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:22.980427 11965 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:08:22.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:22.980829 11965 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:22.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:22.981184 11965 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:22.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:22.981520 11965 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:08:22.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:22.981756 11965 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:22.981913 11965 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34447/jwks.json
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:23.018731 11965 master_runner.cc:386] Master server non-default flags:
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.172.84:43157
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-0/data
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-0/wal
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.172.126:44025,127.11.172.125:40317,127.11.172.124:33559
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.172.126:44025
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34447/jwks.json
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-0/data/info.pb
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.172.126
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-0/logs
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 01:43:20 UTC on e7f111948823
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11719
02:08:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:23.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:23.020040 11965 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:23.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:23.021741 11965 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:23.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:23.033856 11971 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:24.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:23.035279 11972 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:24.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.179165 11974 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:24.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.181357 11973 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1142 milliseconds
02:08:24.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.181490 11965 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:24.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.182806 11965 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:24.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.185829 11965 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:24.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.187294 11965 hybrid_clock.cc:648] HybridClock initialized: now 1777774104187222 us; error 77 us; skew 500 ppm
02:08:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.188215 11965 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:24.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.195163 11965 webserver.cc:492] Webserver started at http://127.11.172.126:46729/ using document root <none> and password file <none>
02:08:24.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.196110 11965 fs_manager.cc:362] Metadata directory not provided
02:08:24.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.196372 11965 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:24.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.196790 11965 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:08:24.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.201455 11965 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-0/data/instance:
02:08:24.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3018e83c4d6b417b9aa645fbfce6471d"
02:08:24.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:24 on dist-test-slave-4s7v"
02:08:24.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.202670 11965 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-0/wal/instance:
02:08:24.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3018e83c4d6b417b9aa645fbfce6471d"
02:08:24.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:24 on dist-test-slave-4s7v"
02:08:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.209987 11965 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
02:08:24.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.216058 11981 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:24.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.217305 11965 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.217693 11965 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-0/data,/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-0/wal
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3018e83c4d6b417b9aa645fbfce6471d"
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:24 on dist-test-slave-4s7v"
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.218055 11965 fs_report.cc:389] FS layout report
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-0/wal
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-0/wal
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-0/data/data
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:24.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.267204 11965 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.268771 11965 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.269281 11965 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:24.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.340281 11965 rpc_server.cc:307] RPC server started. Bound to: 127.11.172.126:44025
02:08:24.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.340358 12032 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.172.126:44025 every 8 connection(s)
02:08:24.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.342967 11965 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-0/data/info.pb
02:08:24.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.346223 11953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu as pid 11965
02:08:24.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.346742 11953 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-0/wal/instance
02:08:24.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.350183 12033 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.351621 11953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-1/wal
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-1/data
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-1/logs
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-1/data/info.pb
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.172.125:40317
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.172.125
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.172.84:43157
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:24.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:24.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.172.126:44025,127.11.172.125:40317,127.11.172.124:33559
02:08:24.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:24.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34447/jwks.json with env {}
02:08:24.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.362802 12033 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } has no permanent_uuid. Determining permanent_uuid...
02:08:24.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.391822 12033 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } has no permanent_uuid. Determining permanent_uuid...
02:08:24.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.394559 11983 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.172.125:40317: connect: Connection refused (error 111)
02:08:24.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.398602 12033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.125:40317: Network error: Client connection negotiation failed: client connection to 127.11.172.125:40317: connect: Connection refused (error 111)
02:08:24.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.448300 12033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } attempt: 1
02:08:24.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.452172 12033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.125:40317: Network error: Client connection negotiation failed: client connection to 127.11.172.125:40317: connect: Connection refused (error 111)
02:08:24.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.520666 12033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } attempt: 2
02:08:24.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.524775 12033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.125:40317: Network error: Client connection negotiation failed: client connection to 127.11.172.125:40317: connect: Connection refused (error 111)
02:08:24.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.616263 12033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } attempt: 3
02:08:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.621416 12033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.125:40317: Network error: Client connection negotiation failed: client connection to 127.11.172.125:40317: connect: Connection refused (error 111)
02:08:24.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.664683 12035 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:24.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.665227 12035 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:24.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.665678 12035 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:24.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.698467 12035 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:08:24.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.698747 12035 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.698962 12035 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.699183 12035 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:08:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.699466 12035 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.699636 12035 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34447/jwks.json
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.737181 12035 master_runner.cc:386] Master server non-default flags:
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.172.84:43157
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-1/data
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-1/wal
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.172.126:44025,127.11.172.125:40317,127.11.172.124:33559
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.172.125:40317
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34447/jwks.json
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-1/data/info.pb
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.172.125
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-1/logs
02:08:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
02:08:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 01:43:20 UTC on e7f111948823
02:08:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11719
02:08:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.738504 12035 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:24.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.740126 12035 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:24.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.750990 12044 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:24.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:24.764860 12033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } attempt: 4
02:08:24.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.770316 12033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.125:40317: Network error: Client connection negotiation failed: client connection to 127.11.172.125:40317: connect: Connection refused (error 111)
02:08:25.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.069883 12033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } attempt: 5
02:08:25.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:25.075376 12033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.125:40317: Network error: Client connection negotiation failed: client connection to 127.11.172.125:40317: connect: Connection refused (error 111)
02:08:25.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.622861 12033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } attempt: 6
02:08:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:25.631275 12033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.125:40317: Network error: Client connection negotiation failed: client connection to 127.11.172.125:40317: connect: Connection refused (error 111)
02:08:25.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:24.752730 12045 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:25.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:25.856081 12047 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:25.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:25.858886 12046 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1101 milliseconds
02:08:25.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.858947 12035 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:25.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.860160 12035 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:25.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.863029 12035 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.864468 12035 hybrid_clock.cc:648] HybridClock initialized: now 1777774105864416 us; error 61 us; skew 500 ppm
02:08:25.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.865267 12035 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:25.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.871536 12035 webserver.cc:492] Webserver started at http://127.11.172.125:39253/ using document root <none> and password file <none>
02:08:25.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.872435 12035 fs_manager.cc:362] Metadata directory not provided
02:08:25.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.872628 12035 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:25.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.873009 12035 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:08:25.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.877349 12035 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-1/data/instance:
02:08:25.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0169d903c7454edaa19db2ccad357851"
02:08:25.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:25 on dist-test-slave-4s7v"
02:08:25.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.878429 12035 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-1/wal/instance:
02:08:25.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0169d903c7454edaa19db2ccad357851"
02:08:25.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:25 on dist-test-slave-4s7v"
02:08:25.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.885510 12035 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
02:08:25.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.891013 12056 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:25.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.892114 12035 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
02:08:25.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.892478 12035 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-1/data,/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-1/wal
02:08:25.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0169d903c7454edaa19db2ccad357851"
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:25 on dist-test-slave-4s7v"
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.892803 12035 fs_report.cc:389] FS layout report
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-1/wal
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-1/wal
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-1/data/data
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:25.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.941741 12035 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.943181 12035 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:25.943600 12035 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:26.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.012050 12035 rpc_server.cc:307] RPC server started. Bound to: 127.11.172.125:40317
02:08:26.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.012146 12107 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.172.125:40317 every 8 connection(s)
02:08:26.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.014993 12035 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-1/data/info.pb
02:08:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.020901 12108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:08:26.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.023567 11953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu as pid 12035
02:08:26.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.024055 11953 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-1/wal/instance
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.027031 11953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-2/wal
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-2/data
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-2/logs
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-2/data/info.pb
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.172.124:33559
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.172.124
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.172.84:43157
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:26.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:26.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.172.126:44025,127.11.172.125:40317,127.11.172.124:33559
02:08:26.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:26.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34447/jwks.json with env {}
02:08:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.035199 12108 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } has no permanent_uuid. Determining permanent_uuid...
02:08:26.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.057569 12108 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } has no permanent_uuid. Determining permanent_uuid...
02:08:26.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.072293 12108 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } has no permanent_uuid. Determining permanent_uuid...
02:08:26.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.074823 12060 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.172.124:33559: connect: Connection refused (error 111)
02:08:26.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.079013 12108 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.124:33559: Network error: Client connection negotiation failed: client connection to 127.11.172.124:33559: connect: Connection refused (error 111)
02:08:26.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.128578 12108 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } attempt: 1
02:08:26.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.132498 12108 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.124:33559: Network error: Client connection negotiation failed: client connection to 127.11.172.124:33559: connect: Connection refused (error 111)
02:08:26.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.200953 12108 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } attempt: 2
02:08:26.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.205091 12108 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.124:33559: Network error: Client connection negotiation failed: client connection to 127.11.172.124:33559: connect: Connection refused (error 111)
02:08:26.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.296469 12108 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } attempt: 3
02:08:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.300941 12108 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.124:33559: Network error: Client connection negotiation failed: client connection to 127.11.172.124:33559: connect: Connection refused (error 111)
02:08:26.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.333302 12110 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:26.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.333930 12110 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:26.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.334470 12110 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.367616 12110 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:08:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.367969 12110 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.368306 12110 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.368588 12110 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:08:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.368857 12110 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.369076 12110 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34447/jwks.json
02:08:26.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.406399 12110 master_runner.cc:386] Master server non-default flags:
02:08:26.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:26.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.172.84:43157
02:08:26.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:26.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-2/data
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-2/wal
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.172.126:44025,127.11.172.125:40317,127.11.172.124:33559
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.172.124:33559
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34447/jwks.json
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-2/data/info.pb
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.172.124
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-2/logs
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:26.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
02:08:26.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:26.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 01:43:20 UTC on e7f111948823
02:08:26.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11719
02:08:26.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:26.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.408059 12110 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:26.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.409773 12110 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:26.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.420727 12120 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.444370 12108 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } attempt: 4
02:08:26.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.448158 12108 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.124:33559: Network error: Client connection negotiation failed: client connection to 127.11.172.124:33559: connect: Connection refused (error 111)
02:08:26.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.691817 12033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } attempt: 7
02:08:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.721460 12033 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } has no permanent_uuid. Determining permanent_uuid...
02:08:26.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.728523 12033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.124:33559: Network error: Client connection negotiation failed: client connection to 127.11.172.124:33559: connect: Connection refused (error 111)
02:08:26.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.747646 12108 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } attempt: 5
02:08:26.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.753345 12108 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.124:33559: Network error: Client connection negotiation failed: client connection to 127.11.172.124:33559: connect: Connection refused (error 111)
02:08:26.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.787055 12033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } attempt: 1
02:08:26.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.792564 12033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.124:33559: Network error: Client connection negotiation failed: client connection to 127.11.172.124:33559: connect: Connection refused (error 111)
02:08:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.874119 12033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } attempt: 2
02:08:26.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.879384 12033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.124:33559: Network error: Client connection negotiation failed: client connection to 127.11.172.124:33559: connect: Connection refused (error 111)
02:08:26.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:26.964900 12033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } attempt: 3
02:08:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.970160 12033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.124:33559: Network error: Client connection negotiation failed: client connection to 127.11.172.124:33559: connect: Connection refused (error 111)
02:08:27.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.110693 12033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } attempt: 4
02:08:27.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:27.116298 12033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.124:33559: Network error: Client connection negotiation failed: client connection to 127.11.172.124:33559: connect: Connection refused (error 111)
02:08:27.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.300884 12108 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } attempt: 6
02:08:27.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:27.310688 12108 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.124:33559: Network error: Client connection negotiation failed: client connection to 127.11.172.124:33559: connect: Connection refused (error 111)
02:08:27.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.399804 12033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } attempt: 5
02:08:27.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:27.404659 12033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.172.124:33559: Network error: Client connection negotiation failed: client connection to 127.11.172.124:33559: connect: Connection refused (error 111)
02:08:27.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:26.421449 12121 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:27.546380 12123 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:27.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:27.550241 12122 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1128 milliseconds
02:08:27.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.550319 12110 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:27.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.551509 12110 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.553916 12110 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:27.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.555258 12110 hybrid_clock.cc:648] HybridClock initialized: now 1777774107555225 us; error 36 us; skew 500 ppm
02:08:27.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.556013 12110 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:27.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.561775 12110 webserver.cc:492] Webserver started at http://127.11.172.124:33551/ using document root <none> and password file <none>
02:08:27.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.562620 12110 fs_manager.cc:362] Metadata directory not provided
02:08:27.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.562808 12110 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:27.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.563184 12110 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:08:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.567581 12110 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-2/data/instance:
02:08:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "be0e7eed63794fe6bf02404226b32ab0"
02:08:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:27 on dist-test-slave-4s7v"
02:08:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.568675 12110 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-2/wal/instance:
02:08:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "be0e7eed63794fe6bf02404226b32ab0"
02:08:27.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:27 on dist-test-slave-4s7v"
02:08:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.575901 12110 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
02:08:27.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.581537 12135 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:27.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.582715 12110 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
02:08:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.583020 12110 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-2/data,/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-2/wal
02:08:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "be0e7eed63794fe6bf02404226b32ab0"
02:08:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:27 on dist-test-slave-4s7v"
02:08:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.583395 12110 fs_report.cc:389] FS layout report
02:08:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-2/wal
02:08:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-2/wal
02:08:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-2/data/data
02:08:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:27.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.639292 12110 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:27.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.640810 12110 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:27.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.641232 12110 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:27.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.710479 12110 rpc_server.cc:307] RPC server started. Bound to: 127.11.172.124:33559
02:08:27.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.710532 12186 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.172.124:33559 every 8 connection(s)
02:08:27.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.713135 12110 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-2/data/info.pb
02:08:27.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.719008 12187 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:08:27.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.721076 11953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu as pid 12110
02:08:27.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.721439 11953 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/master-2/wal/instance
02:08:27.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.732173 12187 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } has no permanent_uuid. Determining permanent_uuid...
02:08:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.763823 12187 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } has no permanent_uuid. Determining permanent_uuid...
02:08:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.773803 12187 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } has no permanent_uuid. Determining permanent_uuid...
02:08:27.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.788939 12187 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0: Bootstrap starting.
02:08:27.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.795682 12187 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0: Neither blocks nor log segments found. Creating new log.
02:08:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.797551 12187 log.cc:826] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0: Log is configured to *not* fsync() on all Append() calls
02:08:27.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.802778 12187 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0: No bootstrap required, opened a new log
02:08:27.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.821811 12187 raft_consensus.cc:359] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } }
02:08:27.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.822592 12187 raft_consensus.cc:385] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:08:27.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.822842 12187 raft_consensus.cc:740] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: be0e7eed63794fe6bf02404226b32ab0, State: Initialized, Role: FOLLOWER
02:08:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.823535 12187 consensus_queue.cc:260] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } }
02:08:27.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.826013 12196 sys_catalog.cc:455] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } } }
02:08:27.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.826581 12196 sys_catalog.cc:458] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [sys.catalog]: This master's current role is: FOLLOWER
02:08:27.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.827451 12187 sys_catalog.cc:565] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [sys.catalog]: configured and running, proceeding with master startup.
02:08:27.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:27.851652 12207 catalog_manager.cc:1568] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:27.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:27.852015 12207 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:27.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.846926 12110 master_runner.cc:186] Error getting master registration for 127.11.172.126:44025: OK, instance_id { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" instance_seqno: 1777774104320930 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:08:27.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.868884 12110 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:08:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:27.896205 12141 tablet.cc:2404] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:08:27.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.957144 12033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } attempt: 6
02:08:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.976940 12033 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d: Bootstrap starting.
02:08:27.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.985005 12033 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d: Neither blocks nor log segments found. Creating new log.
02:08:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.987435 12033 log.cc:826] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d: Log is configured to *not* fsync() on all Append() calls
02:08:27.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:27.992218 12033 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d: No bootstrap required, opened a new log
02:08:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.000686 12196 raft_consensus.cc:493] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:08:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.001255 12196 raft_consensus.cc:515] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } }
02:08:28.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.004604 12196 leader_election.cc:290] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3018e83c4d6b417b9aa645fbfce6471d (127.11.172.126:44025), 0169d903c7454edaa19db2ccad357851 (127.11.172.125:40317)
02:08:28.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.019963 12033 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } }
02:08:28.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.021126 12033 raft_consensus.cc:385] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:08:28.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.021580 12033 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3018e83c4d6b417b9aa645fbfce6471d, State: Initialized, Role: FOLLOWER
02:08:28.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.022614 12033 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } }
02:08:28.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.037528 12211 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } } }
02:08:28.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.038386 12211 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d [sys.catalog]: This master's current role is: FOLLOWER
02:08:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.040081 12033 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d [sys.catalog]: configured and running, proceeding with master startup.
02:08:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.040297 12008 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "be0e7eed63794fe6bf02404226b32ab0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3018e83c4d6b417b9aa645fbfce6471d" is_pre_election: true
02:08:28.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.041216 12008 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate be0e7eed63794fe6bf02404226b32ab0 in term 0.
02:08:28.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.042721 12139 leader_election.cc:304] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3018e83c4d6b417b9aa645fbfce6471d, be0e7eed63794fe6bf02404226b32ab0; no voters:
02:08:28.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.043706 12196 raft_consensus.cc:2804] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:08:28.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.044049 12196 raft_consensus.cc:493] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:08:28.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.044445 12196 raft_consensus.cc:3060] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [term 0 FOLLOWER]: Advancing to term 1
02:08:28.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.059515 12196 raft_consensus.cc:515] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } }
02:08:28.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.062849 12008 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "be0e7eed63794fe6bf02404226b32ab0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3018e83c4d6b417b9aa645fbfce6471d"
02:08:28.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.063489 12008 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d [term 0 FOLLOWER]: Advancing to term 1
02:08:28.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.072151 12008 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate be0e7eed63794fe6bf02404226b32ab0 in term 1.
02:08:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.073304 12139 leader_election.cc:304] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3018e83c4d6b417b9aa645fbfce6471d, be0e7eed63794fe6bf02404226b32ab0; no voters:
02:08:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.076435 12217 raft_consensus.cc:2804] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [term 1 FOLLOWER]: Leader election won for term 1
02:08:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.077526 12196 leader_election.cc:290] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [CANDIDATE]: Term 1 election: Requested vote from peers 3018e83c4d6b417b9aa645fbfce6471d (127.11.172.126:44025), 0169d903c7454edaa19db2ccad357851 (127.11.172.125:40317)
02:08:28.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.081947 12217 raft_consensus.cc:697] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [term 1 LEADER]: Becoming Leader. State: Replica: be0e7eed63794fe6bf02404226b32ab0, State: Running, Role: LEADER
02:08:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.083174 12217 consensus_queue.cc:237] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } }
02:08:28.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.096737 12196 sys_catalog.cc:455] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader be0e7eed63794fe6bf02404226b32ab0. Latest consensus state: current_term: 1 leader_uuid: "be0e7eed63794fe6bf02404226b32ab0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } } }
02:08:28.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.097522 12196 sys_catalog.cc:458] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [sys.catalog]: This master's current role is: LEADER
02:08:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.102924 12225 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
02:08:28.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.111137 12225 catalog_manager.cc:1494] Initializing Kudu cluster ID...
02:08:28.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.145534 12008 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d [term 1 FOLLOWER]: Refusing update from remote peer be0e7eed63794fe6bf02404226b32ab0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:08:28.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.148382 12196 consensus_queue.cc:1048] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:08:28.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.127404 11965 master_runner.cc:186] Error getting master registration for 127.11.172.125:40317: OK, instance_id { permanent_uuid: "0169d903c7454edaa19db2ccad357851" instance_seqno: 1777774105993005 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:08:28.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.158857 11965 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:08:28.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:28.165614 12226 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:28.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:28.166157 12226 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:28.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.202749 12211 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d [sys.catalog]: SysCatalogTable state changed. Reason: New leader be0e7eed63794fe6bf02404226b32ab0. Latest consensus state: current_term: 1 leader_uuid: "be0e7eed63794fe6bf02404226b32ab0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } } }
02:08:28.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.203800 12211 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d [sys.catalog]: This master's current role is: FOLLOWER
02:08:28.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.215021 12219 sys_catalog.cc:455] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "be0e7eed63794fe6bf02404226b32ab0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } } }
02:08:28.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.215904 12219 sys_catalog.cc:458] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [sys.catalog]: This master's current role is: LEADER
02:08:28.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.218540 12211 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "be0e7eed63794fe6bf02404226b32ab0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } } }
02:08:28.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.219646 12211 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d [sys.catalog]: This master's current role is: FOLLOWER
02:08:28.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.221280 12225 catalog_manager.cc:1357] Generated new cluster ID: 10fe8288821d42ddac6ec3005feb689c
02:08:28.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.221581 12225 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
02:08:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.277482 12225 catalog_manager.cc:1380] Generated new certificate authority record
02:08:28.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.279613 12225 catalog_manager.cc:1514] Loading token signing keys...
02:08:28.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.311584 12225 catalog_manager.cc:6044] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0: Generated new TSK 0
02:08:28.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.312819 12225 catalog_manager.cc:1524] Initializing in-progress tserver states...
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.337738 11953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-0/wal
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-0/data
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-0/logs
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-0/data/info.pb
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.172.65:0
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.172.65
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.172.65
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.172.126:44025,127.11.172.125:40317,127.11.172.124:33559
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.172.84:43157
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:08:28.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.371150 12108 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } attempt: 7
02:08:28.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.429636 12108 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851: Bootstrap starting.
02:08:28.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.437347 12108 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851: Neither blocks nor log segments found. Creating new log.
02:08:28.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.439805 12108 log.cc:826] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851: Log is configured to *not* fsync() on all Append() calls
02:08:28.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.461103 12108 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851: No bootstrap required, opened a new log
02:08:28.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.505381 12108 raft_consensus.cc:359] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } }
02:08:28.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.506978 12108 raft_consensus.cc:385] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:08:28.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.507522 12108 raft_consensus.cc:740] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0169d903c7454edaa19db2ccad357851, State: Initialized, Role: FOLLOWER
02:08:28.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.508687 12108 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } }
02:08:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.517503 12237 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } } }
02:08:28.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.522249 12237 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851 [sys.catalog]: This master's current role is: FOLLOWER
02:08:28.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.523339 12081 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851 [term 0 FOLLOWER]: Advancing to term 1
02:08:28.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.522025 12083 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "be0e7eed63794fe6bf02404226b32ab0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0169d903c7454edaa19db2ccad357851" is_pre_election: true
02:08:28.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.528430 12082 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "be0e7eed63794fe6bf02404226b32ab0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0169d903c7454edaa19db2ccad357851"
02:08:28.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.521853 12108 sys_catalog.cc:565] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851 [sys.catalog]: configured and running, proceeding with master startup.
02:08:28.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.532500 12081 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851 [term 1 FOLLOWER]: Refusing update from remote peer be0e7eed63794fe6bf02404226b32ab0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:08:28.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.534303 12219 consensus_queue.cc:1048] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:08:28.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.594568 12237 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851 [sys.catalog]: SysCatalogTable state changed. Reason: New leader be0e7eed63794fe6bf02404226b32ab0. Latest consensus state: current_term: 1 leader_uuid: "be0e7eed63794fe6bf02404226b32ab0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } } }
02:08:28.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.596065 12242 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "be0e7eed63794fe6bf02404226b32ab0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } } }
02:08:28.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.597265 12242 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851 [sys.catalog]: This master's current role is: FOLLOWER
02:08:28.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.597997 12237 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851 [sys.catalog]: This master's current role is: FOLLOWER
02:08:28.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.655699 12253 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:08:28.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:28.882191 12234 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:28.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:28.882726 12234 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:28.883213 12234 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:28.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:28.924502 12234 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:28.925864 12234 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.172.65
02:08:28.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.934204 12219 sys_catalog.cc:455] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "be0e7eed63794fe6bf02404226b32ab0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3018e83c4d6b417b9aa645fbfce6471d" member_type: VOTER last_known_addr { host: "127.11.172.126" port: 44025 } } peers { permanent_uuid: "0169d903c7454edaa19db2ccad357851" member_type: VOTER last_known_addr { host: "127.11.172.125" port: 40317 } } peers { permanent_uuid: "be0e7eed63794fe6bf02404226b32ab0" member_type: VOTER last_known_addr { host: "127.11.172.124" port: 33559 } } }
02:08:28.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.935113 12219 sys_catalog.cc:458] T 00000000000000000000000000000000 P be0e7eed63794fe6bf02404226b32ab0 [sys.catalog]: This master's current role is: LEADER
02:08:28.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.970971 12234 tablet_server_runner.cc:78] Tablet server non-default flags:
02:08:28.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:28.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.172.84:43157
02:08:28.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:28.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:28.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-0/data
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-0/wal
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.172.65:0
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-0/data/info.pb
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.172.65
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.172.126:44025,127.11.172.125:40317,127.11.172.124:33559
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.172.65
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-0/logs
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 01:43:20 UTC on e7f111948823
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11719
02:08:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:28.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.972718 12234 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:28.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:28.974478 12234 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:28.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:28.988678 12261 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:29.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:29.171901 12226 catalog_manager.cc:1269] Loaded cluster ID: 10fe8288821d42ddac6ec3005feb689c
02:08:29.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:29.172195 12226 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d: loading cluster ID for follower catalog manager: success
02:08:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:29.191476 12226 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d: acquiring CA information for follower catalog manager: success
02:08:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:29.196995 12226 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 3018e83c4d6b417b9aa645fbfce6471d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:08:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:29.683161 12253 catalog_manager.cc:1269] Loaded cluster ID: 10fe8288821d42ddac6ec3005feb689c
02:08:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:29.683707 12253 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851: loading cluster ID for follower catalog manager: success
02:08:29.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:29.722129 12253 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851: acquiring CA information for follower catalog manager: success
02:08:29.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:29.751236 12253 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 0169d903c7454edaa19db2ccad357851: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:08:30.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:30.390446 12260 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 12234
02:08:30.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:28.991874 12262 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:30.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:30.991953 12260 kernel_stack_watchdog.cc:198] Thread 12234 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
02:08:30.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:08:30.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:08:30.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:08:30.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:08:30.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:30.993341 12234 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.003s user 0.550s sys 0.898s
02:08:30.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:30.993852 12234 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.004s user 0.551s sys 0.898s
02:08:31.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.000643 12234 server_base.cc:1061] running on GCE node
02:08:31.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:31.002105 12267 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:31.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.003520 12234 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.006072 12234 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:31.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.007542 12234 hybrid_clock.cc:648] HybridClock initialized: now 1777774111007500 us; error 31 us; skew 500 ppm
02:08:31.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.008684 12234 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:31.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.016624 12234 webserver.cc:492] Webserver started at http://127.11.172.65:33315/ using document root <none> and password file <none>
02:08:31.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.017911 12234 fs_manager.cc:362] Metadata directory not provided
02:08:31.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.018217 12234 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:31.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.018795 12234 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:08:31.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.025511 12234 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-0/data/instance:
02:08:31.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6eec537186048d6a5528ef0fe1bcb27"
02:08:31.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:31 on dist-test-slave-4s7v"
02:08:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.027016 12234 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-0/wal/instance:
02:08:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6eec537186048d6a5528ef0fe1bcb27"
02:08:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:31 on dist-test-slave-4s7v"
02:08:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.036937 12234 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.010s sys 0.001s
02:08:31.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.044385 12272 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:31.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.045655 12234 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
02:08:31.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.046066 12234 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-0/data,/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-0/wal
02:08:31.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6eec537186048d6a5528ef0fe1bcb27"
02:08:31.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:31 on dist-test-slave-4s7v"
02:08:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.046499 12234 fs_report.cc:389] FS layout report
02:08:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-0/wal
02:08:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-0/wal
02:08:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-0/data/data
02:08:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:31.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.102996 12234 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:31.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.104591 12234 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:31.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.105065 12234 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:31.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.108064 12234 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:08:31.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.112654 12234 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:08:31.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.112871 12234 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:08:31.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.113179 12234 ts_tablet_manager.cc:616] Registered 0 tablets
02:08:31.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.113361 12234 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:08:31.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.276142 12234 rpc_server.cc:307] RPC server started. Bound to: 127.11.172.65:39289
02:08:31.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.276275 12384 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.172.65:39289 every 8 connection(s)
02:08:31.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.278728 12234 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-0/data/info.pb
02:08:31.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.285745 11953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu as pid 12234
02:08:31.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.286243 11953 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-0/wal/instance
02:08:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.310082 11953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-1/wal
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-1/data
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-1/logs
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-1/data/info.pb
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.172.66:0
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.172.66
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.172.66
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.172.126:44025,127.11.172.125:40317,127.11.172.124:33559
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.172.84:43157
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:08:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.318825 12387 heartbeater.cc:344] Connected to a master server at 127.11.172.126:44025
02:08:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.319407 12387 heartbeater.cc:461] Registering TS with master...
02:08:31.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.320811 12387 heartbeater.cc:507] Master 127.11.172.126:44025 requested a full tablet report, sending...
02:08:31.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.324368 11997 ts_manager.cc:194] Registered new tserver with Master: b6eec537186048d6a5528ef0fe1bcb27 (127.11.172.65:39289)
02:08:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.330646 12385 heartbeater.cc:344] Connected to a master server at 127.11.172.124:33559
02:08:31.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.331007 12385 heartbeater.cc:461] Registering TS with master...
02:08:31.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.331912 12385 heartbeater.cc:507] Master 127.11.172.124:33559 requested a full tablet report, sending...
02:08:31.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.334553 12152 ts_manager.cc:194] Registered new tserver with Master: b6eec537186048d6a5528ef0fe1bcb27 (127.11.172.65:39289)
02:08:31.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.334893 12386 heartbeater.cc:344] Connected to a master server at 127.11.172.125:40317
02:08:31.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.335239 12386 heartbeater.cc:461] Registering TS with master...
02:08:31.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.336126 12386 heartbeater.cc:507] Master 127.11.172.125:40317 requested a full tablet report, sending...
02:08:31.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.337486 12152 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.11.172.65:50435
02:08:31.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.339237 12073 ts_manager.cc:194] Registered new tserver with Master: b6eec537186048d6a5528ef0fe1bcb27 (127.11.172.65:39289)
02:08:31.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:31.630183 12395 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:31.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:31.630707 12395 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:31.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:31.631233 12395 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:31.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:31.666045 12395 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:31.666949 12395 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.172.66
02:08:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.703701 12395 tablet_server_runner.cc:78] Tablet server non-default flags:
02:08:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.172.84:43157
02:08:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-1/data
02:08:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-1/wal
02:08:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.172.66:0
02:08:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-1/data/info.pb
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.172.66
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.172.126:44025,127.11.172.125:40317,127.11.172.124:33559
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.172.66
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-1/logs
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 01:43:20 UTC on e7f111948823
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11719
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.705364 12395 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:31.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:31.707163 12395 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:31.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:31.722502 12402 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:32.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:32.341763 12385 heartbeater.cc:499] Master 127.11.172.124:33559 was elected leader, sending a full tablet report...
02:08:32.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:31.722509 12401 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:32.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:32.886689 12395 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.166s user 0.472s sys 0.685s
02:08:32.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:32.887089 12395 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.167s user 0.472s sys 0.685s
02:08:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:32.891223 12395 server_base.cc:1061] running on GCE node
02:08:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:32.891295 12407 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:32.892650 12395 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:32.894865 12395 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:32.896282 12395 hybrid_clock.cc:648] HybridClock initialized: now 1777774112896202 us; error 42 us; skew 500 ppm
02:08:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:32.897233 12395 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:32.904134 12395 webserver.cc:492] Webserver started at http://127.11.172.66:33535/ using document root <none> and password file <none>
02:08:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:32.905102 12395 fs_manager.cc:362] Metadata directory not provided
02:08:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:32.905326 12395 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:32.905793 12395 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:08:32.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:32.910091 12395 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-1/data/instance:
02:08:32.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "934d1becabe54c21a33bb67aec32aa79"
02:08:32.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:32 on dist-test-slave-4s7v"
02:08:32.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:32.911265 12395 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-1/wal/instance:
02:08:32.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "934d1becabe54c21a33bb67aec32aa79"
02:08:32.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:32 on dist-test-slave-4s7v"
02:08:32.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:32.919437 12395 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.003s
02:08:32.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:32.925801 12413 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:32.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:32.927472 12395 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.002s
02:08:32.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:32.927884 12395 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-1/data,/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-1/wal
02:08:32.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "934d1becabe54c21a33bb67aec32aa79"
02:08:32.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:32 on dist-test-slave-4s7v"
02:08:32.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:32.928284 12395 fs_report.cc:389] FS layout report
02:08:32.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:32.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-1/wal
02:08:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-1/wal
02:08:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-1/data/data
02:08:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:33.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.021948 12395 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:33.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.023473 12395 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:33.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.023936 12395 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:33.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.026818 12395 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:08:33.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.031445 12395 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:08:33.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.031683 12395 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:08:33.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.031986 12395 ts_tablet_manager.cc:616] Registered 0 tablets
02:08:33.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.032205 12395 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:08:33.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.167645 12395 rpc_server.cc:307] RPC server started. Bound to: 127.11.172.66:46389
02:08:33.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.167799 12525 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.172.66:46389 every 8 connection(s)
02:08:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.170790 12395 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-1/data/info.pb
02:08:33.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.173826 11953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu as pid 12395
02:08:33.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.174459 11953 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-1/wal/instance
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.187749 11953 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-2/wal
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-2/data
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-2/logs
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-2/data/info.pb
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.172.67:0
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.172.67
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.172.67
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.172.126:44025,127.11.172.125:40317,127.11.172.124:33559
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.172.84:43157
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:08:33.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.214816 12526 heartbeater.cc:344] Connected to a master server at 127.11.172.124:33559
02:08:33.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.215276 12526 heartbeater.cc:461] Registering TS with master...
02:08:33.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.216718 12526 heartbeater.cc:507] Master 127.11.172.124:33559 requested a full tablet report, sending...
02:08:33.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.217274 12527 heartbeater.cc:344] Connected to a master server at 127.11.172.125:40317
02:08:33.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.217552 12527 heartbeater.cc:461] Registering TS with master...
02:08:33.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.218307 12527 heartbeater.cc:507] Master 127.11.172.125:40317 requested a full tablet report, sending...
02:08:33.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.219600 12152 ts_manager.cc:194] Registered new tserver with Master: 934d1becabe54c21a33bb67aec32aa79 (127.11.172.66:46389)
02:08:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.220862 12152 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.11.172.66:38127
02:08:33.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.222851 12073 ts_manager.cc:194] Registered new tserver with Master: 934d1becabe54c21a33bb67aec32aa79 (127.11.172.66:46389)
02:08:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.228336 12528 heartbeater.cc:344] Connected to a master server at 127.11.172.126:44025
02:08:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.228655 12528 heartbeater.cc:461] Registering TS with master...
02:08:33.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.229209 12528 heartbeater.cc:507] Master 127.11.172.126:44025 requested a full tablet report, sending...
02:08:33.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.231487 11997 ts_manager.cc:194] Registered new tserver with Master: 934d1becabe54c21a33bb67aec32aa79 (127.11.172.66:46389)
02:08:33.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:33.518605 12533 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:33.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:33.519172 12533 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:33.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:33.519661 12533 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:33.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:33.552891 12533 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:33.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:33.553745 12533 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.172.67
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.591787 12533 tablet_server_runner.cc:78] Tablet server non-default flags:
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.172.84:43157
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-2/data
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-2/wal
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.172.67:0
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-2/data/info.pb
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.172.67
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.172.126:44025,127.11.172.125:40317,127.11.172.124:33559
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.172.67
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-2/logs
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 01:43:20 UTC on e7f111948823
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11719
02:08:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:33.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.593380 12533 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:33.594972 12533 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:33.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:33.607267 12542 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.225147 12526 heartbeater.cc:499] Master 127.11.172.124:33559 was elected leader, sending a full tablet report...
02:08:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:33.610417 12543 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:33.613708 12545 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:34.704694 12544 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:08:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.704739 12533 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:34.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.708595 12533 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:34.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.710860 12533 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:34.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.712210 12533 hybrid_clock.cc:648] HybridClock initialized: now 1777774114712159 us; error 65 us; skew 500 ppm
02:08:34.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.713104 12533 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:34.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.719414 12533 webserver.cc:492] Webserver started at http://127.11.172.67:45685/ using document root <none> and password file <none>
02:08:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.720374 12533 fs_manager.cc:362] Metadata directory not provided
02:08:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.720605 12533 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:34.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.721077 12533 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:08:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.725314 12533 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-2/data/instance:
02:08:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d9d14ea97d14395aeb91074dc893eba"
02:08:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:34 on dist-test-slave-4s7v"
02:08:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.726395 12533 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-2/wal/instance:
02:08:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d9d14ea97d14395aeb91074dc893eba"
02:08:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:34 on dist-test-slave-4s7v"
02:08:34.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.733695 12533 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.000s sys 0.008s
02:08:34.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.739391 12554 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.740651 12533 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:08:34.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.740994 12533 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-2/data,/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-2/wal
02:08:34.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d9d14ea97d14395aeb91074dc893eba"
02:08:34.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:34 on dist-test-slave-4s7v"
02:08:34.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.741391 12533 fs_report.cc:389] FS layout report
02:08:34.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:34.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-2/wal
02:08:34.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-2/wal
02:08:34.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-2/data/data
02:08:34.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:34.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:34.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:34.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:34.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:34.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:34.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:34.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:34.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:34.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:34.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:34.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.787875 12533 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:34.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.789507 12533 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:34.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.789958 12533 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:34.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.792546 12533 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:08:34.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.797107 12533 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:08:34.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.797317 12533 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:08:34.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.797523 12533 ts_tablet_manager.cc:616] Registered 0 tablets
02:08:34.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.797667 12533 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:08:34.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.933318 12533 rpc_server.cc:307] RPC server started. Bound to: 127.11.172.67:44457
02:08:34.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.933465 12666 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.172.67:44457 every 8 connection(s)
02:08:34.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.935896 12533 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-2/data/info.pb
02:08:34.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.946561 11953 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu as pid 12533
02:08:34.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:34.947021 11953 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster4234681108071328000/ts-2/wal/instance
02:08:35.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:35.001446 12669 heartbeater.cc:344] Connected to a master server at 127.11.172.126:44025
02:08:35.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:35.001359 12667 heartbeater.cc:344] Connected to a master server at 127.11.172.124:33559
02:08:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:35.001998 12667 heartbeater.cc:461] Registering TS with master...
02:08:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:35.001998 12669 heartbeater.cc:461] Registering TS with master...
02:08:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:35.003551 12669 heartbeater.cc:507] Master 127.11.172.126:44025 requested a full tablet report, sending...
02:08:35.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:35.003552 12667 heartbeater.cc:507] Master 127.11.172.124:33559 requested a full tablet report, sending...
02:08:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:35.006306 12152 ts_manager.cc:194] Registered new tserver with Master: 5d9d14ea97d14395aeb91074dc893eba (127.11.172.67:44457)
02:08:35.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:35.007095 11997 ts_manager.cc:194] Registered new tserver with Master: 5d9d14ea97d14395aeb91074dc893eba (127.11.172.67:44457)
02:08:35.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:35.007812 12152 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.11.172.67:39157
02:08:35.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:35.009423 12668 heartbeater.cc:344] Connected to a master server at 127.11.172.125:40317
02:08:35.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:35.009766 12668 heartbeater.cc:461] Registering TS with master...
02:08:35.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:35.010637 12668 heartbeater.cc:507] Master 127.11.172.125:40317 requested a full tablet report, sending...
02:08:35.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:35.013835 12072 ts_manager.cc:194] Registered new tserver with Master: 5d9d14ea97d14395aeb91074dc893eba (127.11.172.67:44457)
02:08:35.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:35.025784 11953 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
02:08:35.040 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:08:35.040 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
02:08:35.042 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.11.172.126"
port: 44025
}
bound_http_address {
host: "127.11.172.126"
port: 46729
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.11.172.125"
port: 40317
}
bound_http_address {
host: "127.11.172.125"
port: 39253
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.11.172.124"
port: 33559
}
bound_http_address {
host: "127.11.172.124"
port: 33551
}
}
}
02:08:35.042 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:08:35.043 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.11.172.65"
port: 39289
}
bound_http_address {
host: "127.11.172.65"
port: 33315
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.11.172.66"
port: 46389
}
bound_http_address {
host: "127.11.172.66"
port: 33535
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.11.172.67"
port: 44457
}
bound_http_address {
host: "127.11.172.67"
port: 45685
}
}
}
02:08:35.043 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:08:35.044 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster5830590741966415162/krb5kdc/krb5cc
02:08:35.044 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:08:35.045 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskoZkKpr/test-tmp
02:08:35.046 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:08:35.046 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskoZkKpr/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:08:35.049 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
02:08:35.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:35.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:08:35.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:35.373291 12678 mini_oidc.cc:150] Starting JWKS server
02:08:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:35.376627 12678 webserver.cc:492] Webserver started at http://127.0.0.1:36097/ using document root <none> and password file <none>
02:08:35.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:35.377097 12678 mini_oidc.cc:186] Starting OIDC Discovery server
02:08:35.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:35.378760 12678 webserver.cc:492] Webserver started at http://127.0.0.1:39421/ using document root <none> and password file <none>
02:08:35.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
02:08:35.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/principal' for realm 'KRBTEST.COM',
02:08:35.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
02:08:35.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:35 dist-test-slave-4s7v krb5kdc[12689](info): setting up network...
02:08:35.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
02:08:35.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:35 dist-test-slave-4s7v krb5kdc[12689](info): set up 2 sockets
02:08:35.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:35 dist-test-slave-4s7v krb5kdc[12689](info): commencing operation
02:08:35.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
02:08:36.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:36.013510 12667 heartbeater.cc:499] Master 127.11.172.124:33559 was elected leader, sending a full tablet report...
02:08:36.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:36.633248 12522 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad8e0 after lost signal to thread 12396
02:08:36.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:36.634693 12522 debug-util.cc:398] Leaking SignalData structure 0x7b08000add00 after lost signal to thread 12525
02:08:37.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:37.373805 12678 mini_kdc.cc:121] Time spent starting KDC: real 1.960s user 0.007s sys 0.011s
02:08:37.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
02:08:37.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:08:37.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
02:08:37.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
02:08:37.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:08:37.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
02:08:37.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
02:08:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:08:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
02:08:37.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:08:37.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/test-user.keytab.
02:08:37.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/test-user.keytab.
02:08:37.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:37 dist-test-slave-4s7v krb5kdc[12689](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774117, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:37.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:08:37.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-03T02:08:37Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:08:37.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-03T02:08:37Z Disabled control of system clock
02:08:37.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.97.190@KRBTEST.COM; defaulting to no policy
02:08:37.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:37.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.97.190@KRBTEST.COM" created.
02:08:37.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:37.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.97.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab.
02:08:37.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.97.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab.
02:08:37.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.97.190@KRBTEST.COM; defaulting to no policy
02:08:37.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:37.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.97.190@KRBTEST.COM" created.
02:08:37.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:37.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.97.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab.
02:08:37.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.97.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab.
02:08:37.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:37.600489 12678 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:37.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:37.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:08:37.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data
02:08:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/logs
02:08:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data/info.pb
02:08:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.190:41999
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.190
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.190
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:37.916461 12706 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:37.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:37.917004 12706 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:37.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:37.917570 12706 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:37.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:37.951802 12706 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:37.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:37.952093 12706 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:08:37.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:37.952476 12706 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:37.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:37.952732 12706 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:37.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:37.952951 12706 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:08:37.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:37.953164 12706 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:37.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:37.953327 12706 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36097/jwks.json
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:37.992918 12706 master_runner.cc:386] Master server non-default flags:
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.190
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.190:41999
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data/info.pb
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.190
02:08:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/logs
02:08:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
02:08:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 01:43:20 UTC on e7f111948823
02:08:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11719
02:08:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:37.994539 12706 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:37.995519 12706 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:38.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:38.006994 12712 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:39.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:39.299098 12183 debug-util.cc:398] Leaking SignalData structure 0x7b08000af6c0 after lost signal to thread 12115
02:08:39.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:39.299993 12183 debug-util.cc:398] Leaking SignalData structure 0x7b08000b2760 after lost signal to thread 12186
02:08:39.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:38.008005 12713 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:39.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:39.409029 12711 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 12706
02:08:39.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:39.574158 12706 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.567s user 0.459s sys 0.940s
02:08:39.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:39.575596 12706 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.568s user 0.459s sys 0.940s
02:08:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:39.577028 12715 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:39.582096 12714 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1573 milliseconds
02:08:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.582170 12706 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:39.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.583405 12706 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:39.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.586054 12706 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:39.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.587426 12706 hybrid_clock.cc:648] HybridClock initialized: now 1777774119587377 us; error 60 us; skew 500 ppm
02:08:39.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.588393 12706 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:39.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:39 dist-test-slave-4s7v krb5kdc[12689](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774119, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:39.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.594269 12706 init.cc:377] Logged in from keytab as kudu/127.12.97.190@KRBTEST.COM (short username kudu)
02:08:39.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.599869 12706 webserver.cc:492] Webserver started at http://127.12.97.190:40075/ using document root <none> and password file <none>
02:08:39.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.600838 12706 fs_manager.cc:362] Metadata directory not provided
02:08:39.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.601065 12706 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:39.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.601541 12706 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:08:39.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.605924 12706 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data/instance:
02:08:39.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9ed2d8aed72d4d01a0a0549753f10061"
02:08:39.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:39 on dist-test-slave-4s7v"
02:08:39.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.607070 12706 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal/instance:
02:08:39.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9ed2d8aed72d4d01a0a0549753f10061"
02:08:39.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:39 on dist-test-slave-4s7v"
02:08:39.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.618932 12706 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.012s sys 0.000s
02:08:39.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.627048 12724 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:39.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.628216 12706 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.004s
02:08:39.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.628574 12706 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data,/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:08:39.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9ed2d8aed72d4d01a0a0549753f10061"
02:08:39.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:39 on dist-test-slave-4s7v"
02:08:39.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.628893 12706 fs_report.cc:389] FS layout report
02:08:39.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:39.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:08:39.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:08:39.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data/data
02:08:39.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:39.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:39.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:39.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:39.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:39.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:39.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:39.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:39.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:39.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:39.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:39.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.702374 12706 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:39.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.713084 12706 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:39.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.713521 12706 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:39.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.799819 12706 rpc_server.cc:307] RPC server started. Bound to: 127.12.97.190:41999
02:08:39.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.799916 12775 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.97.190:41999 every 8 connection(s)
02:08:39.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.802654 12706 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data/info.pb
02:08:39.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.809221 12776 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:08:39.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.810416 12678 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu as pid 12706
02:08:39.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.811003 12678 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal/instance
02:08:39.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.820933 12776 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } has no permanent_uuid. Determining permanent_uuid...
02:08:39.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.97.189@KRBTEST.COM; defaulting to no policy
02:08:39.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:39.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.97.189@KRBTEST.COM" created.
02:08:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:39 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774119, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.190@KRBTEST.COM for kudu/127.12.97.190@KRBTEST.COM
02:08:39.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.862376 12776 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } has no permanent_uuid. Determining permanent_uuid...
02:08:39.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:39.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.97.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab.
02:08:39.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.97.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab.
02:08:39.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:39.864941 12727 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.97.189:43459: connect: Connection refused (error 111)
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:39.868898 12776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.97.189:43459: Network error: Client connection negotiation failed: client connection to 127.12.97.189:43459: connect: Connection refused (error 111)
02:08:39.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.97.189@KRBTEST.COM; defaulting to no policy
02:08:39.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:39.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.97.189@KRBTEST.COM" created.
02:08:39.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:39.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.97.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab.
02:08:39.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.97.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab.
02:08:39.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.908313 12678 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:39.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:39.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/logs
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data/info.pb
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.189:43459
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.189
02:08:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:39.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:08:39.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.189
02:08:39.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:39.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:39.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:39.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:39.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:08:39.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:39.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:39.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:39.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:39.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:08:39.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:39.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:08:39.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:39.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:39.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.918395 12776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } attempt: 1
02:08:39.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:39.922988 12776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.97.189:43459: Network error: Client connection negotiation failed: client connection to 127.12.97.189:43459: connect: Connection refused (error 111)
02:08:39.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:39.933230 12663 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9fa0 after lost signal to thread 12537
02:08:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:39.934130 12663 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9b20 after lost signal to thread 12666
02:08:39.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:39.991403 12776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } attempt: 2
02:08:39.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:39.996861 12776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.97.189:43459: Network error: Client connection negotiation failed: client connection to 127.12.97.189:43459: connect: Connection refused (error 111)
02:08:40.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:40.088326 12776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } attempt: 3
02:08:40.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:40.093997 12776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.97.189:43459: Network error: Client connection negotiation failed: client connection to 127.12.97.189:43459: connect: Connection refused (error 111)
02:08:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:40.237437 12776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } attempt: 4
02:08:40.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:40.243757 12776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.97.189:43459: Network error: Client connection negotiation failed: client connection to 127.12.97.189:43459: connect: Connection refused (error 111)
02:08:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:40.376051 12785 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:40.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:40.376899 12785 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:40.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:40.377990 12785 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:40.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:40.438104 12785 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:40.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:40.438452 12785 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:08:40.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:40.438925 12785 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:40.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:40.439275 12785 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:40.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:40.439608 12785 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:08:40.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:40.439922 12785 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:40.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:40.440162 12785 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36097/jwks.json
02:08:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:40.510066 12785 master_runner.cc:386] Master server non-default flags:
02:08:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:08:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data
02:08:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:08:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:08:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:08:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.189
02:08:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.189:43459
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data/info.pb
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.189
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/logs
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 01:43:20 UTC on e7f111948823
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11719
02:08:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:40.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:40.511844 12785 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:40.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:40.513267 12785 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:40.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:40.528473 12791 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:40.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:40.543191 12776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } attempt: 5
02:08:40.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:40.548272 12776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.97.189:43459: Network error: Client connection negotiation failed: client connection to 127.12.97.189:43459: connect: Connection refused (error 111)
02:08:41.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:41.095737 12776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } attempt: 6
02:08:41.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:41.126364 12776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.97.189:43459: Network error: Client connection negotiation failed: client connection to 127.12.97.189:43459: connect: Connection refused (error 111)
02:08:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.186793 12776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } attempt: 7
02:08:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.217897 12776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.97.189:43459: Network error: Client connection negotiation failed: client connection to 127.12.97.189:43459: connect: Connection refused (error 111)
02:08:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:40.531297 12792 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:42.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:41.931479 12790 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 12785
02:08:42.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.313889 12790 kernel_stack_watchdog.cc:198] Thread 12785 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
02:08:42.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:08:42.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:08:42.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:08:42.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:08:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.318840 12785 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.789s user 0.548s sys 0.733s
02:08:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.319114 12793 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1783 milliseconds
02:08:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.319267 12785 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.789s user 0.548s sys 0.733s
02:08:42.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.320863 12794 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:42.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.320964 12785 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:42.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.322367 12785 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:42.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.325269 12785 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:42.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.326666 12785 hybrid_clock.cc:648] HybridClock initialized: now 1777774122326612 us; error 60 us; skew 500 ppm
02:08:42.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.327510 12785 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:42.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:42 dist-test-slave-4s7v krb5kdc[12689](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774122, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:42.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.334120 12785 init.cc:377] Logged in from keytab as kudu/127.12.97.189@KRBTEST.COM (short username kudu)
02:08:42.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.338828 12785 webserver.cc:492] Webserver started at http://127.12.97.189:46711/ using document root <none> and password file <none>
02:08:42.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.339851 12785 fs_manager.cc:362] Metadata directory not provided
02:08:42.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.340126 12785 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:42.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.340716 12785 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:08:42.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.346803 12785 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data/instance:
02:08:42.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b3999b7f03a4542a101f3d5a772aa8a"
02:08:42.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:42 on dist-test-slave-4s7v"
02:08:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.348397 12785 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal/instance:
02:08:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b3999b7f03a4542a101f3d5a772aa8a"
02:08:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:42 on dist-test-slave-4s7v"
02:08:42.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.357992 12785 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.004s sys 0.004s
02:08:42.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.365397 12807 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:42.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.366746 12785 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
02:08:42.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.367136 12785 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data,/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:08:42.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b3999b7f03a4542a101f3d5a772aa8a"
02:08:42.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:42 on dist-test-slave-4s7v"
02:08:42.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.367532 12785 fs_report.cc:389] FS layout report
02:08:42.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:42.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:08:42.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:08:42.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data/data
02:08:42.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:42.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:42.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:42.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:42.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:42.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:42.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:42.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:42.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:42.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:42.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.415818 12785 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:42.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.421391 12785 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.421870 12785 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:42.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.494082 12785 rpc_server.cc:307] RPC server started. Bound to: 127.12.97.189:43459
02:08:42.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.494100 12858 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.97.189:43459 every 8 connection(s)
02:08:42.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.496922 12785 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data/info.pb
02:08:42.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.503017 12859 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:08:42.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.504918 12678 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu as pid 12785
02:08:42.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.505242 12678 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal/instance
02:08:42.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.513464 12859 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } has no permanent_uuid. Determining permanent_uuid...
02:08:42.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.97.188@KRBTEST.COM; defaulting to no policy
02:08:42.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:42.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.97.188@KRBTEST.COM" created.
02:08:42.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:42 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774122, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.189@KRBTEST.COM for kudu/127.12.97.190@KRBTEST.COM
02:08:42.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:42.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.97.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab.
02:08:42.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.97.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab.
02:08:42.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.552834 12859 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } has no permanent_uuid. Determining permanent_uuid...
02:08:42.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.97.188@KRBTEST.COM; defaulting to no policy
02:08:42.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:42.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.97.188@KRBTEST.COM" created.
02:08:42.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:42 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774122, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.189@KRBTEST.COM for kudu/127.12.97.189@KRBTEST.COM
02:08:42.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.579926 12859 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } has no permanent_uuid. Determining permanent_uuid...
02:08:42.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.582425 12809 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:08:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.586491 12859 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.97.188:34755: Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:08:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.97.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab.
02:08:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.97.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab.
02:08:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.588788 12678 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:08:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data
02:08:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/logs
02:08:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data/info.pb
02:08:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.188:34755
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.188
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.188
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.636024 12859 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } attempt: 1
02:08:42.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.642891 12859 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.97.188:34755: Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:08:42.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.711306 12859 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } attempt: 2
02:08:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.715092 12859 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.97.188:34755: Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:08:42.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.806533 12859 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } attempt: 3
02:08:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.810963 12859 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.97.188:34755: Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:08:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.905184 12870 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.905824 12870 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.906471 12870 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:42.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.940330 12870 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:42.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.940603 12870 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:08:42.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.940976 12870 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:42.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.941251 12870 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:42.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.941522 12870 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:08:42.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.941789 12870 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:42.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.941998 12870 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36097/jwks.json
02:08:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.954382 12859 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } attempt: 4
02:08:42.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.958647 12859 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.97.188:34755: Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.983757 12870 master_runner.cc:386] Master server non-default flags:
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.188
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.188:34755
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data/info.pb
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.188
02:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/logs
02:08:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
02:08:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 01:43:20 UTC on e7f111948823
02:08:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11719
02:08:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.985040 12870 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:42.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:42.986017 12870 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:42.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.997294 12877 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:43.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:43.258070 12859 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } attempt: 5
02:08:43.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:43.263176 12859 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.97.188:34755: Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:08:43.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:43.810669 12859 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } attempt: 6
02:08:43.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:43.819206 12859 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.97.188:34755: Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:08:44.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:42.997970 12878 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:44.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:44.182878 12880 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:44.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:44.185879 12879 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1183 milliseconds
02:08:44.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.185974 12870 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.187258 12870 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:44.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.190244 12870 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:44.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.191704 12870 hybrid_clock.cc:648] HybridClock initialized: now 1777774124191628 us; error 99 us; skew 500 ppm
02:08:44.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.192608 12870 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:44.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:44 dist-test-slave-4s7v krb5kdc[12689](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774124, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.198853 12870 init.cc:377] Logged in from keytab as kudu/127.12.97.188@KRBTEST.COM (short username kudu)
02:08:44.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.204788 12870 webserver.cc:492] Webserver started at http://127.12.97.188:33143/ using document root <none> and password file <none>
02:08:44.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.205711 12870 fs_manager.cc:362] Metadata directory not provided
02:08:44.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.205906 12870 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:44.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.206305 12870 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:08:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.210608 12870 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data/instance:
02:08:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a23e9e132e145c3ae139ff82dca3ed7"
02:08:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:44 on dist-test-slave-4s7v"
02:08:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.211707 12870 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal/instance:
02:08:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a23e9e132e145c3ae139ff82dca3ed7"
02:08:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:44 on dist-test-slave-4s7v"
02:08:44.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.219579 12870 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.010s sys 0.000s
02:08:44.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.225206 12891 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:44.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.226285 12870 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
02:08:44.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.226646 12870 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data,/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:08:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a23e9e132e145c3ae139ff82dca3ed7"
02:08:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:44 on dist-test-slave-4s7v"
02:08:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.226962 12870 fs_report.cc:389] FS layout report
02:08:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:08:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:08:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data/data
02:08:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:44.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.281517 12870 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:44.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.289016 12870 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:44.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.289579 12870 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:44.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.308357 12776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } attempt: 8
02:08:44.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:44 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774119, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.190@KRBTEST.COM for kudu/127.12.97.189@KRBTEST.COM
02:08:44.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.347275 12776 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } has no permanent_uuid. Determining permanent_uuid...
02:08:44.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:44.354653 12776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.97.188:34755: Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:08:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.388835 12870 rpc_server.cc:307] RPC server started. Bound to: 127.12.97.188:34755
02:08:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.388911 12945 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.97.188:34755 every 8 connection(s)
02:08:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.391450 12870 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data/info.pb
02:08:44.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.396997 12946 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:08:44.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.399734 12678 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu as pid 12870
02:08:44.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.400156 12678 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal/instance
02:08:44.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.411599 12946 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } has no permanent_uuid. Determining permanent_uuid...
02:08:44.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.420596 12776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } attempt: 1
02:08:44.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:44 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774117, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.97.188@KRBTEST.COM
02:08:44.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:44 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774119, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.190@KRBTEST.COM for kudu/127.12.97.188@KRBTEST.COM
02:08:44.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:44 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774124, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.188@KRBTEST.COM for kudu/127.12.97.190@KRBTEST.COM
02:08:44.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.452456 12946 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } has no permanent_uuid. Determining permanent_uuid...
02:08:44.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.464257 12776 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: Bootstrap starting.
02:08:44.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:44 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774124, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.188@KRBTEST.COM for kudu/127.12.97.189@KRBTEST.COM
02:08:44.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.471313 12776 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: Neither blocks nor log segments found. Creating new log.
02:08:44.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.473464 12776 log.cc:826] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: Log is configured to *not* fsync() on all Append() calls
02:08:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.481004 12946 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } has no permanent_uuid. Determining permanent_uuid...
02:08:44.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:44 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774117, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.97.189@KRBTEST.COM
02:08:44.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.486668 12776 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: No bootstrap required, opened a new log
02:08:44.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:44 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774117, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.97.190@KRBTEST.COM
02:08:44.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:44 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774124, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.188@KRBTEST.COM for kudu/127.12.97.188@KRBTEST.COM
02:08:44.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.514595 12776 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:44.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.515417 12776 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:08:44.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.515718 12776 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9ed2d8aed72d4d01a0a0549753f10061, State: Initialized, Role: FOLLOWER
02:08:44.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.517241 12776 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.525694 12955 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.526558 12955 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: This master's current role is: FOLLOWER
02:08:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.527465 12776 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: configured and running, proceeding with master startup.
02:08:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.535424 12946 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: Bootstrap starting.
02:08:44.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.539216 12678 external_mini_cluster.cc:949] 0 TS(s) registered with all masters
02:08:44.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.543687 12946 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: Neither blocks nor log segments found. Creating new log.
02:08:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.546160 12946 log.cc:826] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: Log is configured to *not* fsync() on all Append() calls
02:08:44.550 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:44.550 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
02:08:44.552 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
02:08:44.552 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
02:08:44.554 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.12.97.190"
port: 41999
}
bound_http_address {
host: "127.12.97.190"
port: 40075
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.12.97.189"
port: 43459
}
bound_http_address {
host: "127.12.97.189"
port: 46711
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.12.97.188"
port: 34755
}
bound_http_address {
host: "127.12.97.188"
port: 33143
}
}
}
02:08:44.554 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:08:44.555 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:44.555 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:08:44.555 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:08:44.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:44.560863 12967 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:44.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:44.561358 12967 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:44.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.563388 12946 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: No bootstrap required, opened a new log
02:08:44.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.593881 12946 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:44.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.594748 12946 raft_consensus.cc:385] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:08:44.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.595043 12946 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5a23e9e132e145c3ae139ff82dca3ed7, State: Initialized, Role: FOLLOWER
02:08:44.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.595937 12946 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:44.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.568331 12706 master_runner.cc:186] Error getting master registration for 127.12.97.189:43459: OK, instance_id { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" instance_seqno: 1777774122474848 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:08:44.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.597491 12706 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:08:44.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.600997 12946 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: configured and running, proceeding with master startup.
02:08:44.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.600594 12970 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:44.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.601390 12970 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: This master's current role is: FOLLOWER
02:08:44.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:44 dist-test-slave-4s7v krb5kdc[12689](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774124, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:44.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:08:44.619 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:44.620 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5cc
02:08:44.622 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:08:44.623 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-user
02:08:44.623 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-user"
}
02:08:44.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.637629 12955 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:08:44.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.638176 12955 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:44.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.641387 12955 leader_election.cc:290] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8b3999b7f03a4542a101f3d5a772aa8a (127.12.97.189:43459), 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755)
02:08:44.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.643378 12920 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9ed2d8aed72d4d01a0a0549753f10061" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" is_pre_election: true
02:08:44.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.644338 12920 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9ed2d8aed72d4d01a0a0549753f10061 in term 0.
02:08:44.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.645830 12726 leader_election.cc:304] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5a23e9e132e145c3ae139ff82dca3ed7, 9ed2d8aed72d4d01a0a0549753f10061; no voters:
02:08:44.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.646757 12955 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:08:44.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.647101 12955 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:08:44.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.647563 12955 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 0 FOLLOWER]: Advancing to term 1
02:08:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.654136 12955 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:44.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.659094 12920 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9ed2d8aed72d4d01a0a0549753f10061" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5a23e9e132e145c3ae139ff82dca3ed7"
02:08:44.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.659715 12920 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [term 0 FOLLOWER]: Advancing to term 1
02:08:44.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:44 dist-test-slave-4s7v krb5kdc[12689](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774124, etypes {rep=17 tkt=17 ses=17}, test-user@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:44.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-user@KRBTEST.COM:
02:08:44.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.677635 12955 leader_election.cc:290] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 1 election: Requested vote from peers 8b3999b7f03a4542a101f3d5a772aa8a (127.12.97.189:43459), 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755)
02:08:44.683 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.685348 12920 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9ed2d8aed72d4d01a0a0549753f10061 in term 1.
02:08:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.691627 12726 leader_election.cc:304] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5a23e9e132e145c3ae139ff82dca3ed7, 9ed2d8aed72d4d01a0a0549753f10061; no voters:
02:08:44.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.692880 12955 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 1 FOLLOWER]: Leader election won for term 1
02:08:44.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:44.693619 12982 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:44.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:44.693938 12982 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:44.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.651299 12870 master_runner.cc:186] Error getting master registration for 127.12.97.189:43459: OK, instance_id { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" instance_seqno: 1777774122474848 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:08:44.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.699608 12955 raft_consensus.cc:697] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 1 LEADER]: Becoming Leader. State: Replica: 9ed2d8aed72d4d01a0a0549753f10061, State: Running, Role: LEADER
02:08:44.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.700374 12870 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:08:44.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.701989 12955 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:44.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.713563 12987 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9ed2d8aed72d4d01a0a0549753f10061. Latest consensus state: current_term: 1 leader_uuid: "9ed2d8aed72d4d01a0a0549753f10061" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:44.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.714376 12987 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: This master's current role is: LEADER
02:08:44.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.717144 12989 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
02:08:44.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.724584 12989 catalog_manager.cc:1494] Initializing Kudu cluster ID...
02:08:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:44.724550 12730 tablet.cc:2404] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:08:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.743433 12920 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [term 1 FOLLOWER]: Refusing update from remote peer 9ed2d8aed72d4d01a0a0549753f10061: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:08:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.745326 12955 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:08:44.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.775642 12970 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9ed2d8aed72d4d01a0a0549753f10061. Latest consensus state: current_term: 1 leader_uuid: "9ed2d8aed72d4d01a0a0549753f10061" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:44.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.776396 12970 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: This master's current role is: FOLLOWER
02:08:44.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.790405 12970 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9ed2d8aed72d4d01a0a0549753f10061" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:44.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.791103 12970 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: This master's current role is: FOLLOWER
02:08:44.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.806046 12955 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9ed2d8aed72d4d01a0a0549753f10061" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:44.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.806783 12955 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: This master's current role is: LEADER
02:08:44.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.811444 12989 catalog_manager.cc:1357] Generated new cluster ID: 47a82ce6268a4f58942f3a6c05120811
02:08:44.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.811779 12989 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
02:08:44.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.849238 12989 catalog_manager.cc:1380] Generated new certificate authority record
02:08:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.855134 12989 catalog_manager.cc:1514] Loading token signing keys...
02:08:44.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.879686 12859 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } attempt: 7
02:08:44.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:44 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774122, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.189@KRBTEST.COM for kudu/127.12.97.188@KRBTEST.COM
02:08:44.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.920527 12989 catalog_manager.cc:6044] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: Generated new TSK 0
02:08:44.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.925541 12989 catalog_manager.cc:1524] Initializing in-progress tserver states...
02:08:44.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.938982 12859 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: Bootstrap starting.
02:08:44.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.947729 12859 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: Neither blocks nor log segments found. Creating new log.
02:08:44.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.950690 12859 log.cc:826] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: Log is configured to *not* fsync() on all Append() calls
02:08:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.962499 12859 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: No bootstrap required, opened a new log
02:08:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.991698 12859 raft_consensus.cc:359] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.992633 12859 raft_consensus.cc:385] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:08:44.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.993053 12859 raft_consensus.cc:740] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8b3999b7f03a4542a101f3d5a772aa8a, State: Initialized, Role: FOLLOWER
02:08:44.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.994139 12859 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:44.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.997838 12998 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:44.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.999302 12859 sys_catalog.cc:565] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: configured and running, proceeding with master startup.
02:08:44.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.999495 12833 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9ed2d8aed72d4d01a0a0549753f10061" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8b3999b7f03a4542a101f3d5a772aa8a"
02:08:45.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:44.999861 12832 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 0 FOLLOWER]: Advancing to term 1
02:08:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:45.000465 12834 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9ed2d8aed72d4d01a0a0549753f10061" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" is_pre_election: true
02:08:45.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:45.004535 12998 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: This master's current role is: FOLLOWER
02:08:45.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:45.006758 12832 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 1 FOLLOWER]: Refusing update from remote peer 9ed2d8aed72d4d01a0a0549753f10061: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:08:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:45.008545 12987 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:08:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:45.098743 12998 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9ed2d8aed72d4d01a0a0549753f10061. Latest consensus state: current_term: 1 leader_uuid: "9ed2d8aed72d4d01a0a0549753f10061" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:45.099501 12998 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: This master's current role is: FOLLOWER
02:08:45.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:45.100706 13006 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9ed2d8aed72d4d01a0a0549753f10061" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:45.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:45.101471 13006 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: This master's current role is: FOLLOWER
02:08:45.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:45.132292 13014 catalog_manager.cc:1269] Loaded cluster ID: 47a82ce6268a4f58942f3a6c05120811
02:08:45.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:45.132586 13014 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: loading cluster ID for follower catalog manager: success
02:08:45.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:45.138741 13014 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: acquiring CA information for follower catalog manager: success
02:08:45.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:45.144740 13014 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:08:45.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:45.585654 13017 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9ed2d8aed72d4d01a0a0549753f10061" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:45.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:45.586661 13017 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: This master's current role is: LEADER
02:08:45.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:45.699574 12982 catalog_manager.cc:1269] Loaded cluster ID: 47a82ce6268a4f58942f3a6c05120811
02:08:45.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:45.700081 12982 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: loading cluster ID for follower catalog manager: success
02:08:45.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:45.708276 12982 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: acquiring CA information for follower catalog manager: success
02:08:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:45.714195 12982 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:08:47.685 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.97.190:41999
02:08:47.685 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:08:47.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:47.687595 12678 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu with pid 12706
02:08:47.710 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:47.711 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.97.189:43459
02:08:47.711 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:08:47.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:47.711607 12678 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu with pid 12785
02:08:47.738 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:47.738 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.97.188:34755
02:08:47.738 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:08:47.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:47.738566 12678 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu with pid 12870
02:08:47.762 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:47.762 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.97.190:41999
02:08:47.762 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:47.763818 12678 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/logs
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data/info.pb
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.190:41999
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.190
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40075
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.190
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:47.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:48.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:48.068375 13023 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:48.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:48.068948 13023 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:48.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:48.069626 13023 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:48.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:48.105192 13023 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:48.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:48.105445 13023 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:08:48.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:48.105727 13023 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:48.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:48.105976 13023 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:48.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:48.106204 13023 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:08:48.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:48.106408 13023 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:48.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:48.106564 13023 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36097/jwks.json
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:48.146786 13023 master_runner.cc:386] Master server non-default flags:
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.190
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.190:41999
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data/info.pb
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.190
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40075
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/logs
02:08:48.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:48.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:48.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:48.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:48.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:48.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
02:08:48.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:48.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 01:43:20 UTC on e7f111948823
02:08:48.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11719
02:08:48.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:48.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:48.148084 13023 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:48.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:48.149207 13023 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:48.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:48.161844 13029 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:48.163363 13030 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:48.164950 13032 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:49.272678 13031 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:08:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.272832 13023 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:49.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.277037 13023 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:49.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.279913 13023 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.281476 13023 hybrid_clock.cc:648] HybridClock initialized: now 1777774129281427 us; error 54 us; skew 500 ppm
02:08:49.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.283244 13023 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:49.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:49 dist-test-slave-4s7v krb5kdc[12689](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774129, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:49.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.291195 13023 init.cc:377] Logged in from keytab as kudu/127.12.97.190@KRBTEST.COM (short username kudu)
02:08:49.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.296800 13023 webserver.cc:492] Webserver started at http://127.12.97.190:40075/ using document root <none> and password file <none>
02:08:49.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.297717 13023 fs_manager.cc:362] Metadata directory not provided
02:08:49.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.297967 13023 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:49.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.306143 13023 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
02:08:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.310700 13043 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:49.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.311798 13023 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
02:08:49.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.312161 13023 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data,/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:08:49.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9ed2d8aed72d4d01a0a0549753f10061"
02:08:49.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:39 on dist-test-slave-4s7v"
02:08:49.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.314273 13023 fs_report.cc:389] FS layout report
02:08:49.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:49.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:08:49.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:08:49.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data/data
02:08:49.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:49.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:49.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:49.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:49.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:49.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:49.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:49.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:49.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:49.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:49.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.394812 13023 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.400714 13023 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.401221 13023 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:49.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.474745 13023 rpc_server.cc:307] RPC server started. Bound to: 127.12.97.190:41999
02:08:49.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.474799 13094 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.97.190:41999 every 8 connection(s)
02:08:49.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.478647 13023 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data/info.pb
02:08:49.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.487262 12678 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu as pid 13023
02:08:49.487 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:49.488 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.97.189:43459
02:08:49.488 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.489732 12678 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/logs
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data/info.pb
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.189:43459
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.189
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46711
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.189
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:49.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.491623 13095 sys_catalog.cc:263] Verifying existing consensus state
02:08:49.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.498162 13095 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: Bootstrap starting.
02:08:49.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.537600 13095 log.cc:826] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: Log is configured to *not* fsync() on all Append() calls
02:08:49.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.553089 13095 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:08:49.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.553850 13095 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: Bootstrap complete.
02:08:49.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.582234 13095 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:49.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.584684 13095 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9ed2d8aed72d4d01a0a0549753f10061, State: Initialized, Role: FOLLOWER
02:08:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.585493 13095 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:49.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.588142 13099 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:49.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.588970 13099 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: This master's current role is: FOLLOWER
02:08:49.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.590117 13095 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: configured and running, proceeding with master startup.
02:08:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.637502 13110 catalog_manager.cc:1269] Loaded cluster ID: 47a82ce6268a4f58942f3a6c05120811
02:08:49.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.638017 13110 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: loading cluster ID for follower catalog manager: success
02:08:49.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:49 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774129, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.190@KRBTEST.COM for kudu/127.12.97.190@KRBTEST.COM
02:08:49.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.645506 13110 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: acquiring CA information for follower catalog manager: success
02:08:49.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.651024 13110 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:08:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.664546 13023 master_runner.cc:186] Error getting master registration for 127.12.97.189:43459: Network error: Client connection negotiation failed: client connection to 127.12.97.189:43459: connect: Connection refused (error 111),
02:08:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.690550 13023 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:08:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:49.829335 13097 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:49.829877 13097 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:49.830518 13097 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:49.864020 13097 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:49.864425 13097 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:08:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:49.864916 13097 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:49.865206 13097 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:49.865419 13097 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:08:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:49.865603 13097 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:49.865756 13097 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36097/jwks.json
02:08:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.907155 13097 master_runner.cc:386] Master server non-default flags:
02:08:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:08:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data
02:08:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:08:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:08:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:08:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.189
02:08:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.189:43459
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data/info.pb
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.189
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46711
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/logs
02:08:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
02:08:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 01:43:20 UTC on e7f111948823
02:08:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11719
02:08:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.908669 13097 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:49.909686 13097 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:49.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:49.921352 13120 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:51.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.011516 13126 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:08:51.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.012329 13126 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:51.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:51.031076 13046 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.97.189:43459: connect: Connection refused (error 111)
02:08:51.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.032295 13126 leader_election.cc:290] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 8b3999b7f03a4542a101f3d5a772aa8a (127.12.97.189:43459), 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755)
02:08:51.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:51.038309 13045 leader_election.cc:336] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755): Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:08:51.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:51.039005 13046 leader_election.cc:336] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 8b3999b7f03a4542a101f3d5a772aa8a (127.12.97.189:43459): Network error: Client connection negotiation failed: client connection to 127.12.97.189:43459: connect: Connection refused (error 111)
02:08:51.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.039543 13046 leader_election.cc:304] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9ed2d8aed72d4d01a0a0549753f10061; no voters: 5a23e9e132e145c3ae139ff82dca3ed7, 8b3999b7f03a4542a101f3d5a772aa8a
02:08:51.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.040369 13126 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
02:08:51.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:49.926497 13123 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:51.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:49.922847 13121 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:51.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:51.225515 13122 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1297 milliseconds
02:08:51.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.225663 13097 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.227181 13097 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:51.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.230419 13097 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:51.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.231879 13097 hybrid_clock.cc:648] HybridClock initialized: now 1777774131231780 us; error 97 us; skew 500 ppm
02:08:51.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.233057 13097 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:51.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:51 dist-test-slave-4s7v krb5kdc[12689](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774131, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:51.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.240845 13097 init.cc:377] Logged in from keytab as kudu/127.12.97.189@KRBTEST.COM (short username kudu)
02:08:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.246616 13097 webserver.cc:492] Webserver started at http://127.12.97.189:46711/ using document root <none> and password file <none>
02:08:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.247865 13097 fs_manager.cc:362] Metadata directory not provided
02:08:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.248168 13097 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:51.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.259436 13097 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.005s sys 0.000s
02:08:51.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.265072 13135 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:51.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.266208 13097 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
02:08:51.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.266572 13097 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data,/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:08:51.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b3999b7f03a4542a101f3d5a772aa8a"
02:08:51.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:42 on dist-test-slave-4s7v"
02:08:51.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.268560 13097 fs_report.cc:389] FS layout report
02:08:51.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:51.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:08:51.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:08:51.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data/data
02:08:51.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:51.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:51.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:51.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:51.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:51.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:51.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:51.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:51.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:51.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:51.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:51.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.324303 13097 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.334672 13097 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.335139 13097 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.420364 13097 rpc_server.cc:307] RPC server started. Bound to: 127.12.97.189:43459
02:08:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.420514 13186 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.97.189:43459 every 8 connection(s)
02:08:51.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.423125 13097 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data/info.pb
02:08:51.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.427281 12678 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu as pid 13097
02:08:51.428 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:51.428 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.97.188:34755
02:08:51.428 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:08:51.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.429834 12678 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:51.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:51.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:08:51.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data
02:08:51.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:51.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:51.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:51.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:51.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:51.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:51.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/logs
02:08:51.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data/info.pb
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.188:34755
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.188
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33143
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.188
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:08:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:51.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:51.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.436578 13187 sys_catalog.cc:263] Verifying existing consensus state
02:08:51.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.444793 13187 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: Bootstrap starting.
02:08:51.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:51.492100 12029 debug-util.cc:398] Leaking SignalData structure 0x7b08000af640 after lost signal to thread 11966
02:08:51.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:51.492884 12029 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad160 after lost signal to thread 12032
02:08:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.495783 13187 log.cc:826] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: Log is configured to *not* fsync() on all Append() calls
02:08:51.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.518359 13187 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:08:51.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.519366 13187 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: Bootstrap complete.
02:08:51.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.554574 13187 raft_consensus.cc:359] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:51.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.557890 13187 raft_consensus.cc:740] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8b3999b7f03a4542a101f3d5a772aa8a, State: Initialized, Role: FOLLOWER
02:08:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.559108 13187 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:51.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.567744 13194 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:51.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.568640 13194 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: This master's current role is: FOLLOWER
02:08:51.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.571617 13187 sys_catalog.cc:565] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: configured and running, proceeding with master startup.
02:08:51.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.688926 13205 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:08:51.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:51 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774131, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.189@KRBTEST.COM for kudu/127.12.97.190@KRBTEST.COM
02:08:51.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:51 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774131, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.189@KRBTEST.COM for kudu/127.12.97.189@KRBTEST.COM
02:08:51.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.827191 13097 master_runner.cc:186] Error getting master registration for 127.12.97.188:34755: Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111),
02:08:51.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:51.852852 13097 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:08:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:51.924443 13189 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:51.925252 13189 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:51.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:51.926286 13189 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:51.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:51.986474 13189 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:51.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:51.986819 13189 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:08:51.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:51.987289 13189 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:51.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:51.987633 13189 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:51.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:51.987957 13189 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:08:51.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:51.988301 13189 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:51.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:51.988560 13189 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36097/jwks.json
02:08:52.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.058439 13189 master_runner.cc:386] Master server non-default flags:
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.188
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.188:34755
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data/info.pb
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.188
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33143
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/logs
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
02:08:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:52.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 01:43:20 UTC on e7f111948823
02:08:52.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11719
02:08:52.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:52.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.060325 13189 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:52.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.061764 13189 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:52.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:52.094681 13216 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.575699 13220 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:08:52.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.576524 13220 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:52.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.598060 13220 leader_election.cc:290] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 8b3999b7f03a4542a101f3d5a772aa8a (127.12.97.189:43459), 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755)
02:08:52.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:52.614835 13045 leader_election.cc:336] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755): Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:08:52.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:52 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774129, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.190@KRBTEST.COM for kudu/127.12.97.189@KRBTEST.COM
02:08:52.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.640667 13162 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9ed2d8aed72d4d01a0a0549753f10061" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" is_pre_election: true
02:08:52.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.641537 13162 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9ed2d8aed72d4d01a0a0549753f10061 in term 1.
02:08:52.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.643034 13046 leader_election.cc:304] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8b3999b7f03a4542a101f3d5a772aa8a, 9ed2d8aed72d4d01a0a0549753f10061; no voters: 5a23e9e132e145c3ae139ff82dca3ed7
02:08:52.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.644091 13220 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 1 FOLLOWER]: Leader pre-election won for term 2
02:08:52.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.644419 13220 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:08:52.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.644726 13220 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 1 FOLLOWER]: Advancing to term 2
02:08:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.654127 13220 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:52.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.661985 13162 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9ed2d8aed72d4d01a0a0549753f10061" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "8b3999b7f03a4542a101f3d5a772aa8a"
02:08:52.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.662533 13162 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 1 FOLLOWER]: Advancing to term 2
02:08:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.671778 13162 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9ed2d8aed72d4d01a0a0549753f10061 in term 2.
02:08:52.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.675662 13220 leader_election.cc:290] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 2 election: Requested vote from peers 8b3999b7f03a4542a101f3d5a772aa8a (127.12.97.189:43459), 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755)
02:08:52.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:52.677176 13045 leader_election.cc:336] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755): Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:08:52.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.679070 13046 leader_election.cc:304] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8b3999b7f03a4542a101f3d5a772aa8a, 9ed2d8aed72d4d01a0a0549753f10061; no voters: 5a23e9e132e145c3ae139ff82dca3ed7
02:08:52.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.680408 13220 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 2 FOLLOWER]: Leader election won for term 2
02:08:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.692468 13220 raft_consensus.cc:697] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 2 LEADER]: Becoming Leader. State: Replica: 9ed2d8aed72d4d01a0a0549753f10061, State: Running, Role: LEADER
02:08:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.693928 13220 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:52.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.701881 13205 catalog_manager.cc:1269] Loaded cluster ID: 47a82ce6268a4f58942f3a6c05120811
02:08:52.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.702147 13205 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: loading cluster ID for follower catalog manager: success
02:08:52.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.720566 13205 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: acquiring CA information for follower catalog manager: success
02:08:52.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.730686 13226 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9ed2d8aed72d4d01a0a0549753f10061. Latest consensus state: current_term: 2 leader_uuid: "9ed2d8aed72d4d01a0a0549753f10061" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:52.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.731462 13226 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: This master's current role is: LEADER
02:08:52.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.732342 13205 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:08:52.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.737795 13229 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
02:08:52.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.760630 13229 catalog_manager.cc:1494] Initializing Kudu cluster ID...
02:08:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.773414 13229 catalog_manager.cc:1269] Loaded cluster ID: 47a82ce6268a4f58942f3a6c05120811
02:08:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.774080 13229 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
02:08:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.779444 13229 catalog_manager.cc:1514] Loading token signing keys...
02:08:52.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.785596 13229 catalog_manager.cc:6055] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: Loaded TSK: 0
02:08:52.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:52.787106 13229 catalog_manager.cc:1524] Initializing in-progress tserver states...
02:08:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.212474 13162 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 2 FOLLOWER]: Refusing update from remote peer 9ed2d8aed72d4d01a0a0549753f10061: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
02:08:53.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.214545 13226 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
02:08:53.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.239392 13232 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9ed2d8aed72d4d01a0a0549753f10061. Latest consensus state: current_term: 2 leader_uuid: "9ed2d8aed72d4d01a0a0549753f10061" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:53.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.240419 13232 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: This master's current role is: FOLLOWER
02:08:53.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:53.250861 13045 consensus_peers.cc:597] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 -> Peer 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755): Couldn't send request to peer 5a23e9e132e145c3ae139ff82dca3ed7. Status: Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:08:53.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.263267 13226 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "9ed2d8aed72d4d01a0a0549753f10061" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:53.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.264045 13226 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: This master's current role is: LEADER
02:08:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.266541 13232 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "9ed2d8aed72d4d01a0a0549753f10061" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:53.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.267311 13232 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: This master's current role is: FOLLOWER
02:08:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:53.483372 13215 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 13189
02:08:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:53.680331 13189 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.584s user 0.560s sys 0.850s
02:08:53.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:53.680963 13189 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.585s user 0.560s sys 0.850s
02:08:53.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:52.098541 13217 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:53.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.690606 13189 server_base.cc:1061] running on GCE node
02:08:53.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:53.691653 13239 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.692907 13189 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:53.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.695809 13189 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:53.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.697243 13189 hybrid_clock.cc:648] HybridClock initialized: now 1777774133697113 us; error 132 us; skew 500 ppm
02:08:53.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.698123 13189 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:53.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:53 dist-test-slave-4s7v krb5kdc[12689](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774133, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.704849 13189 init.cc:377] Logged in from keytab as kudu/127.12.97.188@KRBTEST.COM (short username kudu)
02:08:53.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.717312 13189 webserver.cc:492] Webserver started at http://127.12.97.188:33143/ using document root <none> and password file <none>
02:08:53.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.718374 13189 fs_manager.cc:362] Metadata directory not provided
02:08:53.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.718633 13189 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:53.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.727501 13189 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.001s
02:08:53.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.734124 13245 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:53.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.735297 13189 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.002s sys 0.003s
02:08:53.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.735639 13189 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data,/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:08:53.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a23e9e132e145c3ae139ff82dca3ed7"
02:08:53.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:44 on dist-test-slave-4s7v"
02:08:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.737784 13189 fs_report.cc:389] FS layout report
02:08:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:08:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:08:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data/data
02:08:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:53.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.799638 13189 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:53.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.807478 13189 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:53.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.808060 13189 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:53.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.910218 13189 rpc_server.cc:307] RPC server started. Bound to: 127.12.97.188:34755
02:08:53.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.910362 13296 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.97.188:34755 every 8 connection(s)
02:08:53.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.914194 13189 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data/info.pb
02:08:53.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.923403 12678 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu as pid 13189
02:08:53.924 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:53.933 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.97.190:41999(127.12.97.190:41999)] connecting to peer
02:08:53.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.933406 13297 sys_catalog.cc:263] Verifying existing consensus state
02:08:53.939 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.97.189:43459(127.12.97.189:43459)] connecting to peer
02:08:53.940 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.12.97.190:41999(127.12.97.190:41999)] Successfully connected to peer
02:08:53.944 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.97.188:34755(127.12.97.188:34755)] connecting to peer
02:08:53.945 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.12.97.189:43459(127.12.97.189:43459)] Successfully connected to peer
02:08:53.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:53.948796 13297 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: Bootstrap starting.
02:08:53.955 [DEBUG - kudu-nio-1] (SecurityContext.java:222) Refreshing Kerberos credentials...
02:08:53.955 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5cc
02:08:53.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:53 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1777774124, etypes {rep=17 tkt=17 ses=17}, test-user@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:53.960 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-user@KRBTEST.COM
02:08:53.960 [ERROR - kudu-nio-1] (SecurityContext.java:246) Attempted to refresh Kerberos credentials from ticket cache but found that the new Kerberos principal test-user@KRBTEST.COM did not match the original principal test-admin@KRBTEST.COM. Ignoring.
02:08:53.961 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.12.97.189
02:08:53.969 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.12.97.188:34755(127.12.97.188:34755)] Successfully connected to peer
02:08:53.970 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.12.97.190
02:08:53.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:53 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1777774124, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.97.190@KRBTEST.COM
02:08:54.006 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.97.189:43459(127.12.97.189:43459)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
02:08:54.007 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.97.189:43459: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
02:08:54.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:54.010524 13304 negotiation.cc:336] Failed RPC negotiation. Trace:
02:08:54.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:53.945437 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.0.0.1:43290 (local address 127.12.97.189:43459)
02:08:54.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:53.953806 (+ 8369us) server_negotiation.cc:207] Beginning negotiation
02:08:54.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:53.953835 (+ 29us) server_negotiation.cc:400] Waiting for connection header
02:08:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:53.953888 (+ 53us) server_negotiation.cc:408] Connection header received
02:08:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:53.954039 (+ 151us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
02:08:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:53.954068 (+ 29us) server_negotiation.cc:462] Received NEGOTIATE request from client
02:08:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:53.954292 (+ 224us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
02:08:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:53.958975 (+ 4683us) server_negotiation.cc:227] Negotiated authn=SASL
02:08:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:53.966187 (+ 7212us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
02:08:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:53.967740 (+ 1553us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
02:08:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:54.003762 (+ 36022us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
02:08:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:54.004378 (+ 616us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
02:08:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:54.008584 (+ 4206us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:43290: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:43290): Cannot send after transport endpoint shutdown (error 108)
02:08:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":7683,"thread_start_us":7404,"threads_started":1}
02:08:54.017 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.12.97.189:43459(127.12.97.189:43459)] handling channelInactive
02:08:54.023 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd444ed31, L:/127.0.0.1:58186 - R:/127.12.97.190:41999] using SASL/GSSAPI
02:08:54.032 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
02:08:54.032 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=108), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.97.190:41999, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.97.189:43459, [12ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.97.188:34755, [75ms] Sub RPC ConnectToMaster: received response from server master-127.12.97.189:43459: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [99ms] Sub RPC ConnectToMaster: received response from server master-127.12.97.190:41999: OK, deferred=null) after lookup
02:08:54.035 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.12.97.188:34755(127.12.97.188:34755)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
02:08:54.036 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.12.97.188:34755: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
02:08:54.037 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.12.97.188:34755(127.12.97.188:34755)] handling channelInactive
02:08:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:54.040347 13305 negotiation.cc:336] Failed RPC negotiation. Trace:
02:08:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:53.970455 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.0.0.1:41220 (local address 127.12.97.188:34755)
02:08:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:53.985798 (+ 15343us) server_negotiation.cc:207] Beginning negotiation
02:08:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:53.985834 (+ 36us) server_negotiation.cc:400] Waiting for connection header
02:08:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:53.985927 (+ 93us) server_negotiation.cc:408] Connection header received
02:08:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:54.022543 (+ 36616us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
02:08:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:54.022624 (+ 81us) server_negotiation.cc:462] Received NEGOTIATE request from client
02:08:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:54.034923 (+ 12299us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
02:08:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:54.035437 (+ 514us) server_negotiation.cc:227] Negotiated authn=SASL
02:08:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:08:54.038530 (+ 3093us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:41220: BlockingRecv error: recv got EOF from 127.0.0.1:41220 (error 108)
02:08:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":14574,"thread_start_us":29633,"threads_started":1}
02:08:54.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.077685 13297 log.cc:826] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: Log is configured to *not* fsync() on all Append() calls
02:08:54.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.098702 13297 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:08:54.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.099602 13297 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: Bootstrap complete.
02:08:54.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.121718 13297 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:54.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.124094 13297 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5a23e9e132e145c3ae139ff82dca3ed7, State: Initialized, Role: FOLLOWER
02:08:54.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.124926 13297 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:54.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.127702 13307 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:54.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.128612 13307 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: This master's current role is: FOLLOWER
02:08:54.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.129566 13297 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: configured and running, proceeding with master startup.
02:08:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.161939 13318 catalog_manager.cc:1269] Loaded cluster ID: 47a82ce6268a4f58942f3a6c05120811
02:08:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.162364 13318 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: loading cluster ID for follower catalog manager: success
02:08:54.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.169629 13318 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: acquiring CA information for follower catalog manager: success
02:08:54.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:54 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774133, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.188@KRBTEST.COM for kudu/127.12.97.190@KRBTEST.COM
02:08:54.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.176761 13318 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:08:54.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.238457 13272 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [term 1 FOLLOWER]: Advancing to term 2
02:08:54.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.253297 13324 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "9ed2d8aed72d4d01a0a0549753f10061" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:54.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.257558 13324 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: This master's current role is: FOLLOWER
02:08:54.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.258227 13307 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9ed2d8aed72d4d01a0a0549753f10061. Latest consensus state: current_term: 2 leader_uuid: "9ed2d8aed72d4d01a0a0549753f10061" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:54.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.258728 13307 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: This master's current role is: FOLLOWER
02:08:54.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.607415 13220 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "9ed2d8aed72d4d01a0a0549753f10061" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:54.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:54.609287 13220 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: This master's current role is: LEADER
02:08:55.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:55.828629 12663 debug-util.cc:398] Leaking SignalData structure 0x7b08000adc60 after lost signal to thread 12537
02:08:55.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:55.829789 12663 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad3e0 after lost signal to thread 12666
02:08:56.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:56.132583 12522 debug-util.cc:398] Leaking SignalData structure 0x7b08000b56a0 after lost signal to thread 12396
02:08:56.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:56.133680 12522 debug-util.cc:398] Leaking SignalData structure 0x7b08000b54e0 after lost signal to thread 12525
02:08:57.103 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.97.190:41999
02:08:57.103 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:08:57.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:57.104104 12678 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu with pid 13023
02:08:57.129 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:57.129 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.97.189:43459
02:08:57.129 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:08:57.129 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.12.97.190:41999(127.12.97.190:41999)] handling channelInactive
02:08:57.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.97.190:41999(127.12.97.190:41999)] cleaning up while in state READY due to: connection closed
02:08:57.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:57.129700 12678 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu with pid 13097
02:08:57.158 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:57.158 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.97.188:34755
02:08:57.158 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:08:57.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:57.159207 12678 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu with pid 13189
02:08:57.182 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:57.182 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.97.190:41999
02:08:57.182 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:57.184203 12678 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/logs
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data/info.pb
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.190:41999
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.190
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40075
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.190
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:57.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:57.496275 13334 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:57.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:57.496855 13334 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:57.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:57.497510 13334 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:57.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:57.531773 13334 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:57.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:57.532042 13334 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:08:57.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:57.532449 13334 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:57.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:57.532728 13334 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:57.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:57.533017 13334 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:08:57.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:57.533316 13334 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:57.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:57.533535 13334 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36097/jwks.json
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:57.572717 13334 master_runner.cc:386] Master server non-default flags:
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.190
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.190:41999
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data/info.pb
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.190
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40075
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:57.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/logs
02:08:57.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:57.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:57.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:57.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:57.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:57.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
02:08:57.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:57.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 01:43:20 UTC on e7f111948823
02:08:57.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11719
02:08:57.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:57.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:57.574452 13334 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:57.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:57.575680 13334 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:57.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:57.587646 13340 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:58.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:57.589011 13341 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:58.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:57.592199 13343 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:58.718998 13342 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:08:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.719590 13334 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:58.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.723274 13334 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:58.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.726001 13334 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:58.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.727413 13334 hybrid_clock.cc:648] HybridClock initialized: now 1777774138727362 us; error 47 us; skew 500 ppm
02:08:58.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.728300 13334 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:58.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:58 dist-test-slave-4s7v krb5kdc[12689](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774138, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:58.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.734760 13334 init.cc:377] Logged in from keytab as kudu/127.12.97.190@KRBTEST.COM (short username kudu)
02:08:58.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.740645 13334 webserver.cc:492] Webserver started at http://127.12.97.190:40075/ using document root <none> and password file <none>
02:08:58.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.741860 13334 fs_manager.cc:362] Metadata directory not provided
02:08:58.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.742082 13334 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:58.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.750584 13334 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
02:08:58.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.755745 13352 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:58.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.756958 13334 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:08:58.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.757380 13334 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data,/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:08:58.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9ed2d8aed72d4d01a0a0549753f10061"
02:08:58.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:39 on dist-test-slave-4s7v"
02:08:58.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.759377 13334 fs_report.cc:389] FS layout report
02:08:58.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:08:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:08:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data/data
02:08:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.816131 13334 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:58.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.826231 13334 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:58.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.826745 13334 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:58.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.896596 13334 rpc_server.cc:307] RPC server started. Bound to: 127.12.97.190:41999
02:08:58.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.896723 13403 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.97.190:41999 every 8 connection(s)
02:08:58.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.899441 13334 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data/info.pb
02:08:58.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.907895 12678 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu as pid 13334
02:08:58.908 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:58.908 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.97.189:43459
02:08:58.908 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:08:58.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.910351 12678 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:58.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:08:58.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:08:58.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data
02:08:58.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:58.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:58.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:58.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:58.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:58.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/logs
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data/info.pb
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.189:43459
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.189
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46711
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.189
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.910629 13404 sys_catalog.cc:263] Verifying existing consensus state
02:08:58.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.918252 13404 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: Bootstrap starting.
02:08:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.957456 13404 log.cc:826] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: Log is configured to *not* fsync() on all Append() calls
02:08:58.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.976719 13404 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:08:58.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:58.977576 13404 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: Bootstrap complete.
02:08:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:59.003715 13404 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:59.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:59.007846 13404 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9ed2d8aed72d4d01a0a0549753f10061, State: Initialized, Role: FOLLOWER
02:08:59.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:59.008846 13404 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:08:59.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:59.011521 13409 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:08:59.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:59.012379 13409 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: This master's current role is: FOLLOWER
02:08:59.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:59.013252 13404 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: configured and running, proceeding with master startup.
02:08:59.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:59.067174 13420 catalog_manager.cc:1269] Loaded cluster ID: 47a82ce6268a4f58942f3a6c05120811
02:08:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:59.072403 13420 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: loading cluster ID for follower catalog manager: success
02:08:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:59.082990 13420 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: acquiring CA information for follower catalog manager: success
02:08:59.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:08:59 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774138, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.190@KRBTEST.COM for kudu/127.12.97.190@KRBTEST.COM
02:08:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:59.105302 13420 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:08:59.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:59.116418 13334 master_runner.cc:186] Error getting master registration for 127.12.97.189:43459: Network error: Client connection negotiation failed: client connection to 127.12.97.189:43459: connect: Connection refused (error 111),
02:08:59.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:59.151561 13334 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:08:59.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:59.269399 13406 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:59.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:59.269934 13406 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:59.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:59.270572 13406 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:59.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:59.306165 13406 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:59.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:59.306417 13406 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:08:59.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:59.306686 13406 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:59.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:59.306885 13406 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:59.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:59.307077 13406 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:08:59.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:59.307301 13406 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:59.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:59.307474 13406 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36097/jwks.json
02:08:59.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:59.349774 13406 master_runner.cc:386] Master server non-default flags:
02:08:59.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:59.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:08:59.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:59.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:59.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data
02:08:59.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:08:59.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:59.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:59.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:08:59.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:08:59.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.189
02:08:59.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:59.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.189:43459
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data/info.pb
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.189
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46711
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/logs
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 01:43:20 UTC on e7f111948823
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11719
02:08:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:59.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:59.351043 13406 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:59.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:08:59.352093 13406 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:59.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:59.367022 13429 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:09:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:00.840785 13437 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:09:00.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:00.846369 13437 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:09:00.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:00.876641 13437 leader_election.cc:290] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 8b3999b7f03a4542a101f3d5a772aa8a (127.12.97.189:43459), 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755)
02:09:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:00.879423 13355 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.97.189:43459: connect: Connection refused (error 111)
02:09:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:00.899848 13355 leader_election.cc:336] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 8b3999b7f03a4542a101f3d5a772aa8a (127.12.97.189:43459): Network error: Client connection negotiation failed: client connection to 127.12.97.189:43459: connect: Connection refused (error 111)
02:09:00.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:00.902921 13354 leader_election.cc:336] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755): Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:09:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:00.903666 13354 leader_election.cc:304] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9ed2d8aed72d4d01a0a0549753f10061; no voters: 5a23e9e132e145c3ae139ff82dca3ed7, 8b3999b7f03a4542a101f3d5a772aa8a
02:09:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:00.905359 13437 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
02:09:00.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:00.766407 13428 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 13406
02:09:00.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:00.977264 13428 kernel_stack_watchdog.cc:198] Thread 13406 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 395ms:
02:09:00.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:09:00.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:09:00.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:09:00.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:09:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:08:59.369341 13430 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:09:00.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:00.980814 13432 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:09:00.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:00.980051 13406 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.610s user 0.613s sys 0.965s
02:09:00.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:00.981729 13406 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.613s user 0.614s sys 0.966s
02:09:00.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:00.986367 13431 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1616 milliseconds
02:09:00.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:00.986946 13406 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:09:00.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:00.988436 13406 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:09:00.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:00.991539 13406 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:09:00.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:00.992957 13406 hybrid_clock.cc:648] HybridClock initialized: now 1777774140992832 us; error 137 us; skew 500 ppm
02:09:00.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:00.993769 13406 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:09:00.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:09:00 dist-test-slave-4s7v krb5kdc[12689](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774140, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:09:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.000452 13406 init.cc:377] Logged in from keytab as kudu/127.12.97.189@KRBTEST.COM (short username kudu)
02:09:01.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.006357 13406 webserver.cc:492] Webserver started at http://127.12.97.189:46711/ using document root <none> and password file <none>
02:09:01.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.007639 13406 fs_manager.cc:362] Metadata directory not provided
02:09:01.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.007948 13406 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:09:01.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.019466 13406 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.003s sys 0.003s
02:09:01.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.025594 13446 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:09:01.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.026997 13406 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
02:09:01.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.027457 13406 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data,/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:09:01.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b3999b7f03a4542a101f3d5a772aa8a"
02:09:01.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:42 on dist-test-slave-4s7v"
02:09:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.030218 13406 fs_report.cc:389] FS layout report
02:09:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:09:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:09:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:09:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data/data
02:09:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:09:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:09:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:09:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:09:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:09:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:09:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:09:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:09:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:09:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:09:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:09:01.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.094424 13406 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:09:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.100188 13406 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:09:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.100767 13406 kserver.cc:163] Server-wide thread pool size limit: 3276
02:09:01.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.196723 13498 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.97.189:43459 every 8 connection(s)
02:09:01.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.196714 13406 rpc_server.cc:307] RPC server started. Bound to: 127.12.97.189:43459
02:09:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.200814 13406 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data/info.pb
02:09:01.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.207715 12678 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu as pid 13406
02:09:01.208 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:09:01.208 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.97.188:34755
02:09:01.208 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:09:01.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.210165 12678 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/logs
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data/info.pb
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.188:34755
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.188
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33143
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.188
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:09:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:09:01.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:09:01.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:09:01.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:09:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.216070 13499 sys_catalog.cc:263] Verifying existing consensus state
02:09:01.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.221815 13499 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: Bootstrap starting.
02:09:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.260367 13499 log.cc:826] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: Log is configured to *not* fsync() on all Append() calls
02:09:01.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.277335 13499 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:09:01.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.278211 13499 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: Bootstrap complete.
02:09:01.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.309839 13499 raft_consensus.cc:359] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:09:01.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.313102 13499 raft_consensus.cc:740] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8b3999b7f03a4542a101f3d5a772aa8a, State: Initialized, Role: FOLLOWER
02:09:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.314079 13499 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:09:01.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.317157 13503 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:09:01.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.318148 13503 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: This master's current role is: FOLLOWER
02:09:01.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.318907 13499 sys_catalog.cc:565] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: configured and running, proceeding with master startup.
02:09:01.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.361193 13514 catalog_manager.cc:1269] Loaded cluster ID: 47a82ce6268a4f58942f3a6c05120811
02:09:01.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.365703 13514 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: loading cluster ID for follower catalog manager: success
02:09:01.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.383730 13514 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: acquiring CA information for follower catalog manager: success
02:09:01.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:09:01 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774140, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.189@KRBTEST.COM for kudu/127.12.97.190@KRBTEST.COM
02:09:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.396884 13514 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:09:01.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.454054 13406 master_runner.cc:186] Error getting master registration for 127.12.97.188:34755: Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111),
02:09:01.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.480371 13406 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:09:01.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:01.664780 13501 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:09:01.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:01.665567 13501 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:09:01.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:01.666628 13501 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:09:01.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:01.727075 13501 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:09:01.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:01.727409 13501 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:09:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:01.727888 13501 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:09:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:01.728251 13501 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:09:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:01.728588 13501 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:09:01.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:01.728927 13501 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:09:01.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:01.729177 13501 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36097/jwks.json
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.798998 13501 master_runner.cc:386] Master server non-default flags:
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.188
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.188:34755
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:09:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:09:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:09:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:09:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:09:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:09:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:09:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:09:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data/info.pb
02:09:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.188
02:09:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33143
02:09:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:09:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:09:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:09:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:09:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:09:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/logs
02:09:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:09:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:09:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:09:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:09:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:09:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
02:09:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:09:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 01:43:20 UTC on e7f111948823
02:09:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11719
02:09:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:09:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.800760 13501 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:09:01.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:01.802186 13501 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:09:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:01.816428 13525 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:09:02.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.600147 13529 raft_consensus.cc:493] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:09:02.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.601979 13529 raft_consensus.cc:515] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:09:02.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.606807 13379 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "9ed2d8aed72d4d01a0a0549753f10061" is_pre_election: true
02:09:02.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.607594 13379 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8b3999b7f03a4542a101f3d5a772aa8a in term 2.
02:09:02.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.609383 13447 leader_election.cc:304] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8b3999b7f03a4542a101f3d5a772aa8a, 9ed2d8aed72d4d01a0a0549753f10061; no voters:
02:09:02.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:02.618075 13448 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:09:02.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.619453 13529 leader_election.cc:290] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 9ed2d8aed72d4d01a0a0549753f10061 (127.12.97.190:41999), 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755)
02:09:02.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.620625 13529 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 2 FOLLOWER]: Leader pre-election won for term 3
02:09:02.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.621923 13529 raft_consensus.cc:493] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:09:02.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.622762 13529 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 2 FOLLOWER]: Advancing to term 3
02:09:02.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:02.657110 13448 leader_election.cc:336] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755): Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:09:02.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.661149 13529 raft_consensus.cc:515] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:09:02.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.664877 13379 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "9ed2d8aed72d4d01a0a0549753f10061"
02:09:02.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.665541 13379 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 2 FOLLOWER]: Advancing to term 3
02:09:02.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.676826 13529 leader_election.cc:290] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [CANDIDATE]: Term 3 election: Requested vote from peers 9ed2d8aed72d4d01a0a0549753f10061 (127.12.97.190:41999), 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755)
02:09:02.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.680356 13379 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8b3999b7f03a4542a101f3d5a772aa8a in term 3.
02:09:02.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:02.681104 13448 leader_election.cc:336] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755): Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:09:02.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.682363 13447 leader_election.cc:304] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8b3999b7f03a4542a101f3d5a772aa8a, 9ed2d8aed72d4d01a0a0549753f10061; no voters: 5a23e9e132e145c3ae139ff82dca3ed7
02:09:02.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.684307 13529 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 3 FOLLOWER]: Leader election won for term 3
02:09:02.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.685343 13529 raft_consensus.cc:697] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 3 LEADER]: Becoming Leader. State: Replica: 8b3999b7f03a4542a101f3d5a772aa8a, State: Running, Role: LEADER
02:09:02.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.686815 13529 consensus_queue.cc:237] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:09:02.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.706749 13531 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8b3999b7f03a4542a101f3d5a772aa8a. Latest consensus state: current_term: 3 leader_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:09:02.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.708026 13531 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: This master's current role is: LEADER
02:09:02.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.715986 13535 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
02:09:02.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.725831 13535 catalog_manager.cc:1494] Initializing Kudu cluster ID...
02:09:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.730795 13535 catalog_manager.cc:1269] Loaded cluster ID: 47a82ce6268a4f58942f3a6c05120811
02:09:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.731417 13535 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
02:09:02.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.736872 13535 catalog_manager.cc:1514] Loading token signing keys...
02:09:02.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.741503 13535 catalog_manager.cc:6055] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: Loaded TSK: 0
02:09:02.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:02.743227 13535 catalog_manager.cc:1524] Initializing in-progress tserver states...
02:09:03.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:03.114378 13448 consensus_peers.cc:597] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a -> Peer 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755): Couldn't send request to peer 5a23e9e132e145c3ae139ff82dca3ed7. Status: Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:09:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.172107 13379 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 3 FOLLOWER]: Refusing update from remote peer 8b3999b7f03a4542a101f3d5a772aa8a: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
02:09:03.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.181510 13531 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [LEADER]: Connected to new peer: Peer: permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
02:09:03.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:01.818545 13526 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:09:03.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:03.220950 13501 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.404s user 0.005s sys 0.007s
02:09:03.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:03.221346 13501 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.404s user 0.005s sys 0.007s
02:09:03.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.221773 13537 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8b3999b7f03a4542a101f3d5a772aa8a. Latest consensus state: current_term: 3 leader_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:09:03.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.222535 13537 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: This master's current role is: FOLLOWER
02:09:03.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:03.238574 13540 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:09:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.241312 13501 server_base.cc:1061] running on GCE node
02:09:03.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.242707 13501 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:09:03.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.242712 13537 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:09:03.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.243429 13537 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: This master's current role is: FOLLOWER
02:09:03.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.246919 13541 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:09:03.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.247426 13541 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: This master's current role is: LEADER
02:09:03.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.252274 13501 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:09:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.253751 13501 hybrid_clock.cc:648] HybridClock initialized: now 1777774143253656 us; error 96 us; skew 500 ppm
02:09:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.254922 13501 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:09:03.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:09:03 dist-test-slave-4s7v krb5kdc[12689](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774143, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:09:03.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.263944 13501 init.cc:377] Logged in from keytab as kudu/127.12.97.188@KRBTEST.COM (short username kudu)
02:09:03.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.270401 13501 webserver.cc:492] Webserver started at http://127.12.97.188:33143/ using document root <none> and password file <none>
02:09:03.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.271625 13501 fs_manager.cc:362] Metadata directory not provided
02:09:03.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.271916 13501 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:09:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.294770 13501 fs_manager.cc:714] Time spent opening directory manager: real 0.018s user 0.008s sys 0.008s
02:09:03.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.300618 13551 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:09:03.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.302011 13501 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
02:09:03.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.302421 13501 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data,/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:09:03.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a23e9e132e145c3ae139ff82dca3ed7"
02:09:03.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:44 on dist-test-slave-4s7v"
02:09:03.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.305383 13501 fs_report.cc:389] FS layout report
02:09:03.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:09:03.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:09:03.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:09:03.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data/data
02:09:03.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:09:03.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:09:03.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:09:03.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:09:03.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:09:03.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:09:03.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:09:03.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:09:03.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:09:03.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:09:03.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:09:03.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.371044 13501 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:09:03.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.378708 13501 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:09:03.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.379284 13501 kserver.cc:163] Server-wide thread pool size limit: 3276
02:09:03.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.470136 13501 rpc_server.cc:307] RPC server started. Bound to: 127.12.97.188:34755
02:09:03.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.470230 13602 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.97.188:34755 every 8 connection(s)
02:09:03.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.475092 13501 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data/info.pb
02:09:03.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.478593 12678 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu as pid 13501
02:09:03.479 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:09:03.480 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.97.190:41999(127.12.97.190:41999)] connecting to peer
02:09:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.491842 13603 sys_catalog.cc:263] Verifying existing consensus state
02:09:03.494 [DEBUG - kudu-nio-3] (Connection.java:757) [peer master-127.12.97.190:41999(127.12.97.190:41999)] Successfully connected to peer
02:09:03.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.506953 13603 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: Bootstrap starting.
02:09:03.529 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x25bd9b60, L:/127.0.0.1:40882 - R:/127.12.97.190:41999] using TOKEN/null
02:09:03.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.566077 13603 log.cc:826] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: Log is configured to *not* fsync() on all Append() calls
02:09:03.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.583153 13603 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:09:03.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.584026 13603 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: Bootstrap complete.
02:09:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.609617 13603 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:09:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.611984 13603 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5a23e9e132e145c3ae139ff82dca3ed7, State: Initialized, Role: FOLLOWER
02:09:03.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.613281 13603 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:09:03.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.617739 13610 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:09:03.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.618522 13610 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: This master's current role is: FOLLOWER
02:09:03.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.619549 13603 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: configured and running, proceeding with master startup.
02:09:03.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.648473 13621 catalog_manager.cc:1269] Loaded cluster ID: 47a82ce6268a4f58942f3a6c05120811
02:09:03.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.648772 13621 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: loading cluster ID for follower catalog manager: success
02:09:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.656982 13621 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: acquiring CA information for follower catalog manager: success
02:09:03.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.662712 13621 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:09:03.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.721390 13578 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [term 2 FOLLOWER]: Advancing to term 3
02:09:03.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.749217 13630 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:09:03.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.750149 13630 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: This master's current role is: FOLLOWER
02:09:03.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.755134 13610 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8b3999b7f03a4542a101f3d5a772aa8a. Latest consensus state: current_term: 3 leader_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:09:03.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:03.755834 13610 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: This master's current role is: FOLLOWER
02:09:04.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:04.131716 13546 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:09:04.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:04.132547 13546 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: This master's current role is: LEADER
02:09:06.584 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.97.190:41999
02:09:06.585 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:09:06.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:06.585588 12678 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu with pid 13334
02:09:06.609 [DEBUG - kudu-nio-3] (Connection.java:239) [peer master-127.12.97.190:41999(127.12.97.190:41999)] handling channelInactive
02:09:06.609 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.12.97.190:41999(127.12.97.190:41999)] cleaning up while in state READY due to: connection closed
02:09:06.610 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:09:06.610 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.97.189:43459
02:09:06.610 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:09:06.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:06.610641 12678 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu with pid 13406
02:09:06.635 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:09:06.635 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.97.188:34755
02:09:06.635 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:09:06.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:06.636145 12678 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu with pid 13501
02:09:06.662 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:09:06.662 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.97.190:41999
02:09:06.662 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:06.664021 12678 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/logs
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data/info.pb
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.190:41999
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.190
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40075
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.190
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:09:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:09:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:09:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:09:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:09:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:09:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:09:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:09:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:09:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:09:06.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:06.972958 13638 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:09:06.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:06.973567 13638 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:09:06.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:06.974315 13638 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:09:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:06.984444 12104 debug-util.cc:398] Leaking SignalData structure 0x7b08000a5140 after lost signal to thread 12039
02:09:06.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:06.985214 12104 debug-util.cc:398] Leaking SignalData structure 0x7b08000a1d40 after lost signal to thread 12107
02:09:07.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:07.012590 13638 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:09:07.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:07.012925 13638 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:09:07.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:07.013464 13638 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:09:07.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:07.013820 13638 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:09:07.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:07.014153 13638 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:09:07.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:07.014489 13638 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:09:07.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:07.014739 13638 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36097/jwks.json
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:07.084126 13638 master_runner.cc:386] Master server non-default flags:
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.190
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.190:41999
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data/info.pb
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.190
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40075
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/logs
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 01:43:20 UTC on e7f111948823
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11719
02:09:07.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:09:07.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:07.085857 13638 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:09:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:07.087213 13638 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:09:07.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:07.104265 13646 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:09:08.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:07.104393 13645 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:09:08.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:07.107640 13648 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:09:08.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:08.515261 13647 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1408 milliseconds
02:09:08.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.515446 13638 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:09:08.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.516934 13638 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:09:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.519990 13638 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:09:08.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.521425 13638 hybrid_clock.cc:648] HybridClock initialized: now 1777774148521387 us; error 53 us; skew 500 ppm
02:09:08.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.522262 13638 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:09:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:09:08 dist-test-slave-4s7v krb5kdc[12689](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774148, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:09:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.528630 13638 init.cc:377] Logged in from keytab as kudu/127.12.97.190@KRBTEST.COM (short username kudu)
02:09:08.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.535635 13638 webserver.cc:492] Webserver started at http://127.12.97.190:40075/ using document root <none> and password file <none>
02:09:08.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.536710 13638 fs_manager.cc:362] Metadata directory not provided
02:09:08.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.536967 13638 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:09:08.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.545742 13638 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.008s sys 0.000s
02:09:08.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.550561 13656 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:09:08.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.551710 13638 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.552069 13638 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data,/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9ed2d8aed72d4d01a0a0549753f10061"
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:39 on dist-test-slave-4s7v"
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.554083 13638 fs_report.cc:389] FS layout report
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/wal
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data/data
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:09:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:09:08.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.619520 13638 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:09:08.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.632212 13638 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:09:08.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.632866 13638 kserver.cc:163] Server-wide thread pool size limit: 3276
02:09:08.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.716506 13638 rpc_server.cc:307] RPC server started. Bound to: 127.12.97.190:41999
02:09:08.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.716563 13707 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.97.190:41999 every 8 connection(s)
02:09:08.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.720539 13638 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-0/data/info.pb
02:09:08.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.721685 12678 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu as pid 13638
02:09:08.722 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:09:08.722 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.97.189:43459
02:09:08.723 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.724872 12678 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/logs
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data/info.pb
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:09:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.189:43459
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.189
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46711
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.189
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:09:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:09:08.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.735129 13708 sys_catalog.cc:263] Verifying existing consensus state
02:09:08.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.743268 13708 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: Bootstrap starting.
02:09:08.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.797726 13708 log.cc:826] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: Log is configured to *not* fsync() on all Append() calls
02:09:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.820773 13708 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:09:08.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.821789 13708 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: Bootstrap complete.
02:09:08.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.856572 13708 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:09:08.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.859827 13708 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9ed2d8aed72d4d01a0a0549753f10061, State: Initialized, Role: FOLLOWER
02:09:08.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.860888 13708 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:09:08.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.863842 13712 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:09:08.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.864851 13712 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: This master's current role is: FOLLOWER
02:09:08.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.865654 13708 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: configured and running, proceeding with master startup.
02:09:08.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.914191 13723 catalog_manager.cc:1269] Loaded cluster ID: 47a82ce6268a4f58942f3a6c05120811
02:09:08.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.915467 13723 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: loading cluster ID for follower catalog manager: success
02:09:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.925467 13723 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: acquiring CA information for follower catalog manager: success
02:09:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.931813 13723 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:09:08.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:09:08 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774148, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.190@KRBTEST.COM for kudu/127.12.97.190@KRBTEST.COM
02:09:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.957855 13638 master_runner.cc:186] Error getting master registration for 127.12.97.189:43459: Network error: Client connection negotiation failed: client connection to 127.12.97.189:43459: connect: Connection refused (error 111),
02:09:08.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:08.984207 13638 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:09:09.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:09.157270 13710 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:09:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:09.157863 13710 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:09:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:09.158486 13710 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:09:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:09.192860 13710 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:09:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:09.193089 13710 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:09:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:09.193351 13710 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:09:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:09.193548 13710 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:09:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:09.193773 13710 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:09:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:09.194002 13710 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:09:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:09.194248 13710 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36097/jwks.json
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:09.235653 13710 master_runner.cc:386] Master server non-default flags:
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.189
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.189:43459
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:09:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data/info.pb
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.189
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46711
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/logs
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 01:43:20 UTC on e7f111948823
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11719
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:09.237190 13710 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:09.238190 13710 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:09:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:09.249934 13733 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:09:10.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.163980 13739 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:09:10.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.164803 13739 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:09:10.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.184638 13739 leader_election.cc:290] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 8b3999b7f03a4542a101f3d5a772aa8a (127.12.97.189:43459), 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755)
02:09:10.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:10.194310 13658 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:09:10.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:10.202346 13658 leader_election.cc:336] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755): Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:09:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:10.205186 13659 leader_election.cc:336] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 8b3999b7f03a4542a101f3d5a772aa8a (127.12.97.189:43459): Network error: Client connection negotiation failed: client connection to 127.12.97.189:43459: connect: Connection refused (error 111)
02:09:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.205629 13659 leader_election.cc:304] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9ed2d8aed72d4d01a0a0549753f10061; no voters: 5a23e9e132e145c3ae139ff82dca3ed7, 8b3999b7f03a4542a101f3d5a772aa8a
02:09:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.206410 13739 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:09:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:10.498168 12522 debug-util.cc:398] Leaking SignalData structure 0x7b08000b56e0 after lost signal to thread 12396
02:09:10.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:10.499261 12522 debug-util.cc:398] Leaking SignalData structure 0x7b08000ada40 after lost signal to thread 12525
02:09:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:09.254189 13736 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:09:10.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:10.587877 13735 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1336 milliseconds
02:09:10.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.587920 13710 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:09:10.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:09.252224 13734 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:09:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.592707 13710 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:09:10.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.595619 13710 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:09:10.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.597057 13710 hybrid_clock.cc:648] HybridClock initialized: now 1777774150596885 us; error 184 us; skew 500 ppm
02:09:10.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.597962 13710 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:09:10.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:09:10 dist-test-slave-4s7v krb5kdc[12689](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774150, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:09:10.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.604645 13710 init.cc:377] Logged in from keytab as kudu/127.12.97.189@KRBTEST.COM (short username kudu)
02:09:10.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.610110 13710 webserver.cc:492] Webserver started at http://127.12.97.189:46711/ using document root <none> and password file <none>
02:09:10.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.611042 13710 fs_manager.cc:362] Metadata directory not provided
02:09:10.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.611264 13710 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:09:10.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.619808 13710 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.002s sys 0.004s
02:09:10.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.624562 13748 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:09:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.625702 13710 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.626036 13710 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data,/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b3999b7f03a4542a101f3d5a772aa8a"
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:42 on dist-test-slave-4s7v"
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.628046 13710 fs_report.cc:389] FS layout report
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/wal
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data/data
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:09:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:09:10.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.676012 13710 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:09:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.681948 13710 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:09:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.682462 13710 kserver.cc:163] Server-wide thread pool size limit: 3276
02:09:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.768219 13710 rpc_server.cc:307] RPC server started. Bound to: 127.12.97.189:43459
02:09:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.768324 13800 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.97.189:43459 every 8 connection(s)
02:09:10.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.771211 13710 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-1/data/info.pb
02:09:10.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.777810 12678 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu as pid 13710
02:09:10.778 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:09:10.778 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.97.188:34755
02:09:10.778 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:09:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.779872 12678 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:09:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu
02:09:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:09:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data
02:09:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:09:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:09:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:09:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:09:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:09:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:09:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/logs
02:09:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data/info.pb
02:09:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:09:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:09:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:09:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:09:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:09:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.188:34755
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.188
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33143
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.188
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:09:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:09:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.786758 13801 sys_catalog.cc:263] Verifying existing consensus state
02:09:10.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.795089 13801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: Bootstrap starting.
02:09:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.851593 13801 log.cc:826] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: Log is configured to *not* fsync() on all Append() calls
02:09:10.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.877523 13801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:09:10.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.878539 13801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: Bootstrap complete.
02:09:10.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.913872 13801 raft_consensus.cc:359] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:09:10.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.917338 13801 raft_consensus.cc:740] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8b3999b7f03a4542a101f3d5a772aa8a, State: Initialized, Role: FOLLOWER
02:09:10.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.918376 13801 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:09:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.921269 13805 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:09:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.922233 13805 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: This master's current role is: FOLLOWER
02:09:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.923278 13801 sys_catalog.cc:565] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: configured and running, proceeding with master startup.
02:09:10.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.961581 13816 catalog_manager.cc:1269] Loaded cluster ID: 47a82ce6268a4f58942f3a6c05120811
02:09:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.966979 13816 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: loading cluster ID for follower catalog manager: success
02:09:10.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.984287 13816 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: acquiring CA information for follower catalog manager: success
02:09:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:09:10 dist-test-slave-4s7v krb5kdc[12689](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774150, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.189@KRBTEST.COM for kudu/127.12.97.190@KRBTEST.COM
02:09:10.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:10.992221 13816 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:09:11.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:11.047235 13710 master_runner.cc:186] Error getting master registration for 127.12.97.188:34755: Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111),
02:09:11.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:11.104225 13710 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:09:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:11.200471 13803 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:09:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:11.201282 13803 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:09:11.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:11.202358 13803 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:09:11.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:11.261560 13803 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:09:11.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:11.261905 13803 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:09:11.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:11.262383 13803 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:09:11.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:11.262742 13803 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:09:11.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:11.263077 13803 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:09:11.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:11.263401 13803 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:09:11.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:11.263665 13803 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36097/jwks.json
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:11.331610 13803 master_runner.cc:386] Master server non-default flags:
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.97.148:34129
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/kudu.keytab
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.97.188
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.97.188:34755
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36097/jwks.json
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data/info.pb
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.97.188
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33143
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:09:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:09:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/logs
02:09:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:09:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:09:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:09:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:09:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:09:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
02:09:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:09:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 01:43:20 UTC on e7f111948823
02:09:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11719
02:09:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:09:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:11.333325 13803 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:09:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:11.334694 13803 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:09:11.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:11.349364 13828 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:09:12.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.074370 13833 raft_consensus.cc:493] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:09:12.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.075588 13833 raft_consensus.cc:515] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:09:12.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.084301 13683 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "9ed2d8aed72d4d01a0a0549753f10061" is_pre_election: true
02:09:12.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.085237 13683 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8b3999b7f03a4542a101f3d5a772aa8a in term 3.
02:09:12.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.087911 13749 leader_election.cc:304] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8b3999b7f03a4542a101f3d5a772aa8a, 9ed2d8aed72d4d01a0a0549753f10061; no voters:
02:09:12.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.104984 13833 leader_election.cc:290] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9ed2d8aed72d4d01a0a0549753f10061 (127.12.97.190:41999), 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755)
02:09:12.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.113164 13833 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 3 FOLLOWER]: Leader pre-election won for term 4
02:09:12.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.116005 13833 raft_consensus.cc:493] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:09:12.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.118685 13833 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 3 FOLLOWER]: Advancing to term 4
02:09:12.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:12.129746 13750 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:09:12.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:12.134122 13749 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.12.97.190:41999, user_credentials={real_user=slave}} blocked reactor thread for 46527.6us
02:09:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:12.144395 13750 leader_election.cc:336] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755): Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:09:12.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.151028 13833 raft_consensus.cc:515] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:09:12.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.153945 13683 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "9ed2d8aed72d4d01a0a0549753f10061"
02:09:12.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.154510 13683 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 3 FOLLOWER]: Advancing to term 4
02:09:12.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:12.172418 13750 leader_election.cc:336] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755): Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111)
02:09:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.170328 13833 leader_election.cc:290] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [CANDIDATE]: Term 4 election: Requested vote from peers 9ed2d8aed72d4d01a0a0549753f10061 (127.12.97.190:41999), 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755)
02:09:12.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.174253 13683 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8b3999b7f03a4542a101f3d5a772aa8a in term 4.
02:09:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.176442 13749 leader_election.cc:304] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8b3999b7f03a4542a101f3d5a772aa8a, 9ed2d8aed72d4d01a0a0549753f10061; no voters: 5a23e9e132e145c3ae139ff82dca3ed7
02:09:12.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.177656 13833 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 4 FOLLOWER]: Leader election won for term 4
02:09:12.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.178579 13833 raft_consensus.cc:697] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [term 4 LEADER]: Becoming Leader. State: Replica: 8b3999b7f03a4542a101f3d5a772aa8a, State: Running, Role: LEADER
02:09:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.179751 13833 consensus_queue.cc:237] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:09:12.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.200258 13834 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8b3999b7f03a4542a101f3d5a772aa8a. Latest consensus state: current_term: 4 leader_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:09:12.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.201432 13834 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: This master's current role is: LEADER
02:09:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.211898 13838 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
02:09:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.232033 13838 catalog_manager.cc:1494] Initializing Kudu cluster ID...
02:09:12.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.239147 13838 catalog_manager.cc:1269] Loaded cluster ID: 47a82ce6268a4f58942f3a6c05120811
02:09:12.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.239930 13838 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
02:09:12.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.246418 13838 catalog_manager.cc:1514] Loading token signing keys...
02:09:12.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.252940 13838 catalog_manager.cc:6055] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a: Loaded TSK: 0
02:09:12.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.254797 13838 catalog_manager.cc:1524] Initializing in-progress tserver states...
02:09:12.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:11.351807 13829 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:09:12.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:12.640854 13803 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.294s user 0.522s sys 0.690s
02:09:12.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:12.641525 13803 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.294s user 0.522s sys 0.691s
02:09:12.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.645969 13803 server_base.cc:1061] running on GCE node
02:09:12.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:12.646927 13840 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:09:12.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.648590 13803 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:09:12.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.651635 13803 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:09:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.655216 13803 hybrid_clock.cc:648] HybridClock initialized: now 1777774152655142 us; error 61 us; skew 500 ppm
02:09:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.656337 13803 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:09:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 02:09:12 dist-test-slave-4s7v krb5kdc[12689](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777774152, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.97.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:09:12.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.668138 13803 init.cc:377] Logged in from keytab as kudu/127.12.97.188@KRBTEST.COM (short username kudu)
02:09:12.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.673930 13683 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [term 4 FOLLOWER]: Refusing update from remote peer 8b3999b7f03a4542a101f3d5a772aa8a: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
02:09:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.675936 13834 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [LEADER]: Connected to new peer: Peer: permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
02:09:12.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.706992 13803 webserver.cc:492] Webserver started at http://127.12.97.188:33143/ using document root <none> and password file <none>
02:09:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.708454 13803 fs_manager.cc:362] Metadata directory not provided
02:09:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.708753 13803 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:09:12.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.703851 13846 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8b3999b7f03a4542a101f3d5a772aa8a. Latest consensus state: current_term: 4 leader_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:09:12.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.719535 13846 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: This master's current role is: FOLLOWER
02:09:12.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.724505 13834 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:09:12.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.725921 13834 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a [sys.catalog]: This master's current role is: LEADER
02:09:12.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.726328 13803 fs_manager.cc:714] Time spent opening directory manager: real 0.013s user 0.006s sys 0.005s
02:09:12.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.723860 13852 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:09:12.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.727317 13852 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed2d8aed72d4d01a0a0549753f10061 [sys.catalog]: This master's current role is: FOLLOWER
02:09:12.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:12.750255 13750 consensus_peers.cc:597] T 00000000000000000000000000000000 P 8b3999b7f03a4542a101f3d5a772aa8a -> Peer 5a23e9e132e145c3ae139ff82dca3ed7 (127.12.97.188:34755): Couldn't send request to peer 5a23e9e132e145c3ae139ff82dca3ed7. Status: Network error: Client connection negotiation failed: client connection to 127.12.97.188:34755: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:09:12.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.751569 13856 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:09:12.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.753137 13803 fs_manager.cc:730] Time spent opening block manager: real 0.024s user 0.006s sys 0.017s
02:09:12.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.753582 13803 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data,/tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:09:12.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a23e9e132e145c3ae139ff82dca3ed7"
02:09:12.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 02:08:44 on dist-test-slave-4s7v"
02:09:12.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.756322 13803 fs_report.cc:389] FS layout report
02:09:12.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:09:12.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:09:12.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/wal
02:09:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data/data
02:09:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:09:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:09:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:09:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:09:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:09:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:09:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:09:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:09:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:09:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:09:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:09:12.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.830220 13803 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:09:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.837399 13803 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:09:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.837898 13803 kserver.cc:163] Server-wide thread pool size limit: 3276
02:09:12.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.933872 13803 rpc_server.cc:307] RPC server started. Bound to: 127.12.97.188:34755
02:09:12.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.933920 13907 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.97.188:34755 every 8 connection(s)
02:09:12.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.937294 13803 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/master-2/data/info.pb
02:09:12.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.946434 12678 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu as pid 13803
02:09:12.947 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:09:12.947 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.97.190:41999(127.12.97.190:41999)] connecting to peer
02:09:12.949 [DEBUG - kudu-nio-4] (Connection.java:757) [peer master-127.12.97.190:41999(127.12.97.190:41999)] Successfully connected to peer
02:09:12.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.954509 13908 sys_catalog.cc:263] Verifying existing consensus state
02:09:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:12.962838 13908 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: Bootstrap starting.
02:09:12.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:12.969911 13911 negotiation.cc:343] Unauthorized connection attempt: Server connection negotiation failed: server connection from 127.0.0.1:39434: token expired
02:09:12.974 [DEBUG - kudu-nio-4] (Negotiator.java:352) peer /127.12.97.190:41999 sent connection negotiation error: Not authorized: token expired
02:09:12.976 [DEBUG - kudu-nio-4] (Connection.java:239) [peer master-127.12.97.190:41999(127.12.97.190:41999)] handling channelInactive
02:09:12.977 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.12.97.190:41999(127.12.97.190:41999)] cleaning up while in state NEGOTIATION_FAILED due to: connection closed
02:09:12.979 [DEBUG - kudu-nio-4] (Connection.java:748) [peer master-127.12.97.190:41999(127.12.97.190:41999)] connecting to peer
02:09:12.980 [DEBUG - kudu-nio-4] (Connection.java:748) [peer master-127.12.97.189:43459(127.12.97.189:43459)] connecting to peer
02:09:12.981 [DEBUG - kudu-nio-5] (Connection.java:757) [peer master-127.12.97.190:41999(127.12.97.190:41999)] Successfully connected to peer
02:09:12.984 [DEBUG - kudu-nio-4] (Connection.java:748) [peer master-127.12.97.188:34755(127.12.97.188:34755)] connecting to peer
02:09:12.987 [DEBUG - kudu-nio-5] (SecurityContext.java:222) Refreshing Kerberos credentials...
02:09:12.988 [DEBUG - kudu-nio-5] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoZkKpr/test-tmp/mini-kudu-cluster17914022838650091143/krb5kdc/krb5cc
02:09:12.989 [DEBUG - kudu-nio-5] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:09:12.989 [WARN - kudu-nio-5] (SecurityContext.java:231) Tried to refresh Kerberos credentials but was unable to re-login from ticket cache
02:09:12.989 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.12.97.190:41999(127.12.97.190:41999)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
02:09:12.990 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.12.97.190:41999: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
02:09:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:12.991971 13911 negotiation.cc:336] Failed RPC negotiation. Trace:
02:09:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:12.981533 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.0.0.1:39444 (local address 127.12.97.190:41999)
02:09:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:12.982180 (+ 647us) server_negotiation.cc:207] Beginning negotiation
02:09:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:12.982229 (+ 49us) server_negotiation.cc:400] Waiting for connection header
02:09:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:12.983547 (+ 1318us) server_negotiation.cc:408] Connection header received
02:09:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:12.985489 (+ 1942us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
02:09:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:12.985532 (+ 43us) server_negotiation.cc:462] Received NEGOTIATE request from client
02:09:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:12.985829 (+ 297us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
02:09:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:12.985999 (+ 170us) server_negotiation.cc:227] Negotiated authn=SASL
02:09:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:12.990672 (+ 4673us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:39444: BlockingRecv error: recv got EOF from 127.0.0.1:39444 (error 108)
02:09:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":93}
02:09:12.993 [DEBUG - kudu-nio-5] (Connection.java:239) [peer master-127.12.97.190:41999(127.12.97.190:41999)] handling channelInactive
02:09:12.994 [DEBUG - kudu-nio-6] (Connection.java:757) [peer master-127.12.97.189:43459(127.12.97.189:43459)] Successfully connected to peer
02:09:12.997 [DEBUG - kudu-nio-7] (Connection.java:757) [peer master-127.12.97.188:34755(127.12.97.188:34755)] Successfully connected to peer
02:09:13.005 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.12.97.189:43459(127.12.97.189:43459)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
02:09:13.006 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.12.97.189:43459: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
02:09:13.006 [DEBUG - kudu-nio-6] (Connection.java:239) [peer master-127.12.97.189:43459(127.12.97.189:43459)] handling channelInactive
02:09:13.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:13.008450 13915 negotiation.cc:336] Failed RPC negotiation. Trace:
02:09:13.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:12.987026 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.0.0.1:44514 (local address 127.12.97.189:43459)
02:09:13.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:12.997335 (+ 10309us) server_negotiation.cc:207] Beginning negotiation
02:09:13.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:12.997363 (+ 28us) server_negotiation.cc:400] Waiting for connection header
02:09:13.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:12.997420 (+ 57us) server_negotiation.cc:408] Connection header received
02:09:13.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:12.997579 (+ 159us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
02:09:13.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:12.997611 (+ 32us) server_negotiation.cc:462] Received NEGOTIATE request from client
02:09:13.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:12.997837 (+ 226us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
02:09:13.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:12.998025 (+ 188us) server_negotiation.cc:227] Negotiated authn=SASL
02:09:13.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:13.006883 (+ 8858us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44514: BlockingRecv error: recv got EOF from 127.0.0.1:44514 (error 108)
02:09:13.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":9732,"thread_start_us":9429,"threads_started":1}
02:09:13.016 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.12.97.188:34755(127.12.97.188:34755)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
02:09:13.016 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.12.97.188:34755: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
02:09:13.018 [DEBUG - main] (Connection.java:575) [peer master-127.12.97.190:41999(127.12.97.190:41999)] disconnecting while in state TERMINATED
02:09:13.018 [DEBUG - main] (Connection.java:575) [peer master-127.12.97.188:34755(127.12.97.188:34755)] disconnecting while in state TERMINATED
02:09:13.018 [DEBUG - main] (Connection.java:575) [peer master-127.12.97.189:43459(127.12.97.189:43459)] disconnecting while in state TERMINATED
02:09:13.019 [DEBUG - kudu-nio-7] (Connection.java:239) [peer master-127.12.97.188:34755(127.12.97.188:34755)] handling channelInactive
02:09:13.019 [DEBUG - kudu-nio-6] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:09:13.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:13.020267 12678 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu with pid 13638
02:09:13.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:13.022545 13908 log.cc:826] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: Log is configured to *not* fsync() on all Append() calls
02:09:13.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:13.026878 13916 negotiation.cc:336] Failed RPC negotiation. Trace:
02:09:13.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:12.998190 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.0.0.1:42444 (local address 127.12.97.188:34755)
02:09:13.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:13.006379 (+ 8189us) server_negotiation.cc:207] Beginning negotiation
02:09:13.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:13.006421 (+ 42us) server_negotiation.cc:400] Waiting for connection header
02:09:13.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:13.008377 (+ 1956us) server_negotiation.cc:408] Connection header received
02:09:13.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:13.014317 (+ 5940us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
02:09:13.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:13.014386 (+ 69us) server_negotiation.cc:462] Received NEGOTIATE request from client
02:09:13.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:13.014916 (+ 530us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
02:09:13.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:13.015420 (+ 504us) server_negotiation.cc:227] Negotiated authn=SASL
02:09:13.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 02:09:13.019461 (+ 4041us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:42444: BlockingRecv error: recv got EOF from 127.0.0.1:42444 (error 108)
02:09:13.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":6764,"thread_start_us":6380,"threads_started":1}
02:09:13.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:13.050240 13908 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:09:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:13.051239 13908 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7: Bootstrap complete.
02:09:13.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:13.058784 12678 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu with pid 13710
02:09:13.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:13.077904 13908 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:09:13.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:13.082338 13908 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5a23e9e132e145c3ae139ff82dca3ed7, State: Initialized, Role: FOLLOWER
02:09:13.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:13.083448 13908 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } }
02:09:13.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:13.097347 13919 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed2d8aed72d4d01a0a0549753f10061" member_type: VOTER last_known_addr { host: "127.12.97.190" port: 41999 } } peers { permanent_uuid: "8b3999b7f03a4542a101f3d5a772aa8a" member_type: VOTER last_known_addr { host: "127.12.97.189" port: 43459 } } peers { permanent_uuid: "5a23e9e132e145c3ae139ff82dca3ed7" member_type: VOTER last_known_addr { host: "127.12.97.188" port: 34755 } } }
02:09:13.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:13.098178 13919 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: This master's current role is: FOLLOWER
02:09:13.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:13.104365 13908 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5a23e9e132e145c3ae139ff82dca3ed7 [sys.catalog]: configured and running, proceeding with master startup.
02:09:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:13.105837 12678 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu with pid 13803
02:09:14.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-03T02:09:14Z chronyd exiting
02:09:15.243 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:09:15.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:15.245182 11953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu with pid 12234
02:09:15.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:15.264809 11953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu with pid 12395
02:09:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:15.287500 11953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu with pid 12533
02:09:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:15.309875 11953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu with pid 11965
02:09:15.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:15.337677 11953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu with pid 12035
02:09:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 02:09:15.365363 12137 connection.cc:570] client connection to 127.11.172.125:40317 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
02:09:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 02:09:15.365756 11953 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoZkKpr/build/tsan/bin/kudu with pid 12110
02:09:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-03T02:09:16Z chronyd exiting
02:09:17.462 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158)
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158)
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
02:09:17.649 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient: giving up after 1 attempts
02:09:17.649 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158)
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.12.97.190:41999,127.12.97.189:43459,127.12.97.188:34755). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158)
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more